def main(): fe = DummyFeatureExtractor() env = EwapGridworld( ped_id=1, vision_radius=4, ) rl = ActorCritic(env, feat_extractor=fe, max_episodes=10**4) rl.train()
def main(): args = parser.parse_args() env = GridWorld(display=args.render, obstacles=[np.asarray([1, 2])], step_wrapper=utils.step_wrapper, reset_wrapper=utils.reset_wrapper, stepReward=.01) model = ActorCritic(env, gamma=0.99, log_interval=100, max_episodes=10**4, max_ep_length=30) if args.policy_path is not None: model.policy.load(args.policy_path) if not args.play: model.train() if not args.dont_save: model.policy.save('./saved-models/') if args.play: env.tickSpeed = 15 assert args.policy_path is not None, 'pass a policy to play from!' model.generate_trajectory(args.num_trajs, './trajs/ac_gridworld/')
def main(): args = parser.parse_args() mp.set_start_method('spawn') env = GridWorld(display=False, obstacles=[np.asarray([1, 2])], reset_wrapper=reset_wrapper, step_wrapper=step_wrapper) model = ActorCritic(env, gamma=0.99, log_interval=100, max_episodes=10**4, max_ep_length=30) if args.policy_path is not None: model.policy.load(args.policy_path) if not args.play: model.train_mp(n_jobs=4) if not args.dont_save: model.policy.save('./saved-models/') if args.play: assert args.policy_path is not None, 'pass a policy to play from!' model.generate_trajectory(args.num_trajs, './trajs/mp_gridworld/')
def main(): args = parser.parse_args() if args.render: from envs.gridworld import GridWorld else: from envs.gridworld_clockless import GridWorldClockless as GridWorld env = GridWorld(display=args.render, obstacles=[np.asarray([1, 2])], goal_state=np.asarray([5, 5]), step_wrapper=step_wrapper, reset_wrapper=reset_wrapper, seed=3) loss_t = LBT(list_size=100, stop_threshold=1.5, log_interval=100) model = ActorCritic(env, gamma=0.99, log_interval=200, max_episodes=5000, max_ep_length=20, termination=loss_t) if args.policy_path is not None: model.policy.load(args.policy_path) if args.reward_net is not None: reward_net = RewardNet(env.reset().shape[0]) reward_net.to('cuda') reward_net.load('./saved-models-rewards/0.pt') reward_net.eval() else: reward_net = None if not args.play: model.train_mp(n_jobs=4, reward_net=reward_net, irl=args.irl) if not args.dont_save: model.policy.save('./saved-models/') if args.play: env.tickSpeed = 15 assert args.policy_path is not None, 'pass a policy to play from!' model.generate_trajectory(args.num_trajs, './trajs/ac_gridworld/')
def main(): args = parser.parse_args() if args.on_server: # matplotlib without monitor matplotlib.use('Agg') # pygame without monitor os.environ['SDL_VIDEODRIVER'] = 'dummy' ### ts = time.time() st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') ### parent_dir = './results/' + str(args.save_folder) + st to_save = './results/'+str(args.save_folder)+st+'-reg-'+str(args.regularizer)+ \ '-seed-'+str(args.seed)+'-lr-'+str(args.lr) #'-seed-'+str(args.seed) log_file = 'Experiment_info.txt' experiment_logger = Logger(to_save, log_file) experiment_logger.log_header('Arguments for the experiment :') experiment_logger.log_info(vars(args)) from rlmethods.rlutils import LossBasedTermination from rlmethods.b_actor_critic import ActorCritic from irlmethods.deep_maxent import DeepMaxEnt import irlmethods.irlUtils as irlUtils #check for the feature extractor being used #initialize feature extractor if args.feat_extractor == 'MCFeatures': feat_ext = MCFeatures(args.state_discretization[0], args.state_discretization[1]) if args.feat_extractor == 'MCFeaturesOnehot': feat_ext = MCFeaturesOnehot(args.state_discretization[0], args.state_discretization[1]) experiment_logger.log_header('Parameters of the feature extractor :') experiment_logger.log_info(feat_ext.__dict__) #initialize the environment if not args.dont_save and args.save_folder is None: print('Specify folder to save the results.') exit() #**set is_onehot to false goal_state = np.asarray([1, 5]) ''' env = GridWorld(display=args.render, is_onehot= False,is_random=False, rows =10, cols =10, seed = 7, obstacles = [np.asarray([5,5])], goal_state = np.asarray([1,5])) ''' env = gym.make('MountainCar-v0') env = env.unwrapped #pdb.set_trace() experiment_logger.log_header('Environment details :') experiment_logger.log_info(env.__dict__) #CHANGE HEREq #CHANGE HERE #initialize loss based termination # intialize RL method #CHANGE HERE rlMethod = ActorCritic(env, gamma=1, log_interval=args.rl_log_intervals, max_episodes=args.rl_episodes, max_ep_length=args.rl_ep_length, termination=None, plot_loss=False, save_folder=to_save, hidden_dims=args.policy_net_hidden_dims, feat_extractor=feat_ext) print("RL method initialized.") print(rlMethod.policy) if args.policy_path is not None: rlMethod.policy.load(args.policy_path) experiment_logger.log_header('Details of the RL method :') experiment_logger.log_info(rlMethod.__dict__) # initialize IRL method #CHANGE HERE trajectory_path = args.exp_trajectory_path irlMethod = DeepMaxEnt(trajectory_path, rlmethod=rlMethod, env=env, iterations=args.irl_iterations, log_intervals=5, on_server=args.on_server, regularizer=args.regularizer, learning_rate=args.lr, graft=False, seed=args.seed, scale_svf=args.scale_svf, hidden_dims=args.reward_net_hidden_dims, clipping_value=args.clipping_value, save_folder=parent_dir) print("IRL method intialized.") experiment_logger.log_header('Details of the IRL method :') experiment_logger.log_info(irlMethod.__dict__) print(irlMethod.reward) rewardNetwork = irlMethod.train() if not args.dont_save: pass
def main(): env = gym.make('CartPole-v0') model = ActorCritic(env, gamma=0.99, log_interval=1, max_ep_length=200) model.train()
def main(): args = parser.parse_args() experiment_logger = Logger('temp_save.txt') experiment_logger.log_header('Arguments for the experiment :') experiment_logger.log_info(vars(args)) mp.set_start_method('spawn') if args.render: from envs.gridworld import GridWorld else: from envs.gridworld_clockless import GridWorldClockless as GridWorld agent_width = 10 step_size = 10 obs_width = 10 grid_size = 10 if args.feat_extractor == 'Onehot': feat_ext = OneHot(grid_rows=10, grid_cols=10) if args.feat_extractor == 'SocialNav': feat_ext = SocialNav(fieldList=['agent_state', 'goal_state']) if args.feat_extractor == 'FrontBackSideSimple': feat_ext = FrontBackSideSimple( thresh1=1, thresh2=2, thresh3=3, thresh4=4, step_size=step_size, agent_width=agent_width, obs_width=obs_width, ) if args.feat_extractor == 'LocalGlobal': feat_ext = LocalGlobal( window_size=3, grid_size=grid_size, agent_width=agent_width, obs_width=obs_width, step_size=step_size, ) experiment_logger.log_header('Parameters of the feature extractor :') experiment_logger.log_info(feat_ext.__dict__) ''' np.asarray([2,2]),np.asarray([7,4]),np.asarray([3,5]), np.asarray([5,2]),np.asarray([8,3]),np.asarray([7,5]), np.asarray([3,3]),np.asarray([3,7]),np.asarray([5,7]) ''' env = GridWorld(display=args.render, is_onehot=False, is_random=True, rows=100, agent_width=agent_width, step_size=step_size, obs_width=obs_width, width=grid_size, cols=100, seed=7, buffer_from_obs=0, obstacles=3, goal_state=np.asarray([5, 5])) experiment_logger.log_header('Environment details :') experiment_logger.log_info(env.__dict__) model = ActorCritic(env, feat_extractor=feat_ext, gamma=0.99, log_interval=100, max_ep_length=40, hidden_dims=args.policy_net_hidden_dims, max_episodes=4000) experiment_logger.log_header('Details of the RL method :') experiment_logger.log_info(model.__dict__) pdb.set_trace() if args.policy_path is not None: model.policy.load(args.policy_path) if not args.play and not args.play_user: if args.reward_path is None: model.train_mp(n_jobs=4) else: from irlmethods.deep_maxent import RewardNet state_size = featExtract.extract_features(env.reset()).shape[0] reward_net = RewardNet(state_size) reward_net.load(args.reward_path) print(next(reward_net.parameters()).is_cuda) model.train_mp(reward_net=reward_net, n_jobs=4) if not args.dont_save: model.policy.save('./saved-models/') if args.play: env.tickSpeed = 15 assert args.policy_path is not None, 'pass a policy to play from!' model.generate_trajectory(args.num_trajs, './trajs/ac_fbs_simple4_static_map7/') if args.play_user: env.tickSpeed = 200 model.generate_trajectory_user(args.num_trajs, './trajs/ac_gridworld_user/')
def main(): args = parser.parse_args() if args.on_server: # matplotlib without monitor matplotlib.use('Agg') # pygame without monitor os.environ['SDL_VIDEODRIVER'] = 'dummy' #####for the logger base_folder = './results/' + str(args.save_folder) + '-reg-' + str( args.regularizer) + '-seed-' + str(args.seed) + '-lr-' + str(args.lr) log_file = 'Experiment_info.txt' experiment_logger = Logger(base_folder, log_file) experiment_logger.log_header('Arguments for the experiment :') experiment_logger.log_info(vars(args)) from rlmethods.rlutils import LossBasedTermination from rlmethods.b_actor_critic import ActorCritic from irlmethods.deep_maxent import DeepMaxEnt import irlmethods.irlUtils as irlUtils from featureExtractor.gridworld_featureExtractor import OneHot, LocalGlobal, SocialNav, FrontBackSideSimple agent_width = 10 step_size = 10 obs_width = 10 grid_size = 10 if args.feat_extractor is None: print('Feature extractor missing.') exit() #check for the feature extractor being used #initialize feature extractor if args.feat_extractor == 'Onehot': feat_ext = OneHot(grid_rows=10, grid_cols=10) if args.feat_extractor == 'SocialNav': feat_ext = SocialNav() if args.feat_extractor == 'FrontBackSideSimple': feat_ext = FrontBackSideSimple( thresh1=1, thresh2=2, thresh3=3, thresh4=4, step_size=step_size, agent_width=agent_width, obs_width=obs_width, ) if args.feat_extractor == 'LocalGlobal': feat_ext = LocalGlobal( window_size=5, grid_size=grid_size, agent_width=agent_width, obs_width=obs_width, step_size=step_size, ) experiment_logger.log_header('Parameters of the feature extractor :') experiment_logger.log_info(feat_ext.__dict__) #initialize the environment if not args.dont_save and args.save_folder is None: print('Specify folder to save the results.') exit() if args.annotation_file is None: print('Specify annotation file for the environment.') exit() if args.exp_trajectory_path is None: print('Specify expert trajectory folder.') exit() #**set is_onehot to false goal_state = np.asarray([1, 5]) ''' env = GridWorld(display=args.render, is_onehot= False,is_random=False, rows =10, cols =10, seed = 7, obstacles = [np.asarray([5,5])], goal_state = np.asarray([1,5])) ''' env = GridWorld(display=args.render, is_random=True, rows=576, cols=720, agent_width=agent_width, step_size=step_size, obs_width=obs_width, width=grid_size, annotation_file=args.annotation_file, goal_state=goal_state, step_wrapper=utils.step_wrapper, seed=args.seed, reset_wrapper=utils.reset_wrapper, is_onehot=False) experiment_logger.log_header('Environment details :') experiment_logger.log_info(env.__dict__) #CHANGE HEREq #CHANGE HERE #initialize loss based termination # intialize RL method #CHANGE HERE rlMethod = ActorCritic(env, gamma=0.99, log_interval=args.rl_log_intervals, max_episodes=args.rl_episodes, max_ep_length=args.rl_ep_length, termination=None, hidden_dims=args.reward_net_hidden_dims, feat_extractor=feat_ext) print("RL method initialized.") print(rlMethod.policy) if args.policy_path is not None: rlMethod.policy.load(args.policy_path) experiment_logger.log_header('Details of the RL method :') experiment_logger.log_info(rlMethod.__dict__) # initialize IRL method #CHANGE HERE trajectory_path = args.exp_trajectory_path folder_to_save = '/results/' + args.save_folder irlMethod = DeepMaxEnt(trajectory_path, rlmethod=rlMethod, env=env, iterations=args.irl_iterations, log_intervals=5, on_server=args.on_server, regularizer=args.regularizer, learning_rate=args.lr, graft=True, hidden_dims=args.reward_net_hidden_dims, save_folder=folder_to_save) print("IRL method intialized.") print(irlMethod.reward) experiment_logger.log_header('Details of the IRL method :') experiment_logger.log_info(irlMethod.__dict__) rewardNetwork = irlMethod.train() if not args.dont_save: pass
def main(): args = parser.parse_args() utils.seed_all(args.seed) if args.on_server: # matplotlib without monitor matplotlib.use("Agg") # pygame without monitor os.environ["SDL_VIDEODRIVER"] = "dummy" #####for the logger ts = time.time() st = datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%d_%H:%M:%S") ################### if not args.save_folder: print("Provide save folder.") exit() policy_net_dims = "-policy_net-" for dim in args.policy_net_hidden_dims: policy_net_dims += str(dim) policy_net_dims += "-" reward_net_dims = "-reward_net-" for dim in args.reward_net_hidden_dims: reward_net_dims += str(dim) reward_net_dims += "-" parent_dir = ("./results/" + str(args.save_folder) + st + policy_net_dims + reward_net_dims) to_save = ("./results/" + str(args.save_folder) + st + policy_net_dims + reward_net_dims + "-reg-" + str(args.regularizer) + "-seed-" + str(args.seed) + "-lr-" + str(args.lr_irl)) log_file = "Experiment_info.txt" experiment_logger = Logger(to_save, log_file) experiment_logger.log_header("Arguments for the experiment :") repo = git.Repo(search_parent_directories=True) experiment_logger.log_info({'From branch : ': repo.active_branch.name}) experiment_logger.log_info({'Commit number : ': repo.head.object.hexsha}) experiment_logger.log_info(vars(args)) # from rlmethods.rlutils import LossBasedTermination # for rl from rlmethods.b_actor_critic import ActorCritic from rlmethods.soft_ac_pi import SoftActorCritic from rlmethods.soft_ac import SoftActorCritic as QSAC from rlmethods.rlutils import ReplayBuffer # for irl from irlmethods.deep_maxent import DeepMaxEnt import irlmethods.irlUtils as irlUtils from featureExtractor.gridworld_featureExtractor import ( OneHot, LocalGlobal, SocialNav, FrontBackSideSimple, ) agent_width = 10 step_size = 2 obs_width = 10 grid_size = 10 if args.feat_extractor is None: print("Feature extractor missing.") exit() # check for the feature extractor being used # initialize feature extractor if args.feat_extractor == "Onehot": feat_ext = OneHot(grid_rows=10, grid_cols=10) if args.feat_extractor == "SocialNav": feat_ext = SocialNav() if args.feat_extractor == "FrontBackSideSimple": feat_ext = FrontBackSideSimple( thresh1=1, thresh2=2, thresh3=3, thresh4=4, step_size=step_size, agent_width=agent_width, obs_width=obs_width, ) if args.feat_extractor == "LocalGlobal": feat_ext = LocalGlobal( window_size=5, grid_size=grid_size, agent_width=agent_width, obs_width=obs_width, step_size=step_size, ) if args.feat_extractor == "DroneFeatureSAM1": feat_ext = DroneFeatureSAM1( agent_width=agent_width, obs_width=obs_width, step_size=step_size, grid_size=grid_size, thresh1=5, thresh2=10, ) if args.feat_extractor == "DroneFeatureRisk": feat_ext = DroneFeatureRisk( agent_width=agent_width, obs_width=obs_width, step_size=step_size, grid_size=grid_size, thresh1=15, thresh2=30, ) if args.feat_extractor == "DroneFeatureRisk_v2": feat_ext = DroneFeatureRisk_v2( agent_width=agent_width, obs_width=obs_width, step_size=step_size, grid_size=grid_size, thresh1=15, thresh2=30, ) if args.feat_extractor == "DroneFeatureRisk_speed": feat_ext = DroneFeatureRisk_speed( agent_width=agent_width, obs_width=obs_width, step_size=step_size, grid_size=grid_size, thresh1=10, thresh2=15, ) if args.feat_extractor == "DroneFeatureRisk_speedv2": feat_ext = DroneFeatureRisk_speedv2( agent_width=agent_width, obs_width=obs_width, step_size=step_size, grid_size=grid_size, thresh1=18, thresh2=30, ) if args.feat_extractor == 'VasquezF1': feat_ext = VasquezF1(agent_width * 6, 0.5, 1.0) if args.feat_extractor == 'VasquezF2': feat_ext = VasquezF1(agent_width * 6, 0.5, 1.0) if args.feat_extractor == 'VasquezF3': feat_ext = VasquezF3(agent_width) if args.feat_extractor == "Fahad": feat_ext = Fahad(36, 60, 0.5, 1.0) if args.feat_extractor == "GoalConditionedFahad": feat_ext = GoalConditionedFahad(36, 60, 0.5, 1.0) experiment_logger.log_header("Parameters of the feature extractor :") experiment_logger.log_info(feat_ext.__dict__) # initialize the environment if not args.dont_save and args.save_folder is None: print("Specify folder to save the results.") exit() """ environment can now initialize without an annotation file if args.annotation_file is None: print('Specify annotation file for the environment.') exit() """ if args.exp_trajectory_path is None: print("Specify expert trajectory folder.") exit() """ env = GridWorld(display=args.render, is_onehot= False,is_random=False, rows =10, cols =10, seed = 7, obstacles = [np.asarray([5,5])], goal_state = np.asarray([1,5])) """ env = GridWorld( display=args.render, is_random=True, rows=576, cols=720, agent_width=agent_width, step_size=step_size, obs_width=obs_width, width=grid_size, subject=args.subject, annotation_file=args.annotation_file, goal_state=None, step_wrapper=utils.step_wrapper, seed=args.seed, replace_subject=args.replace_subject, segment_size=args.segment_size, external_control=True, continuous_action=False, reset_wrapper=utils.reset_wrapper, consider_heading=True, is_onehot=False, ) experiment_logger.log_header("Environment details :") experiment_logger.log_info(env.__dict__) # CHANGE HEREq # CHANGE HERE # initialize loss based termination # intialize RL method # CHANGE HERE if args.rl_method == "ActorCritic": rl_method = ActorCritic( env, feat_extractor=feat_ext, gamma=1, log_interval=args.rl_log_intervals, max_episode_length=args.rl_ep_length, hidden_dims=args.policy_net_hidden_dims, save_folder=to_save, lr=args.lr_rl, max_episodes=args.rl_episodes, ) if args.rl_method == "SAC": if not env.continuous_action: print("The action space needs to be continuous for SAC to work.") exit() replay_buffer = ReplayBuffer(args.replay_buffer_size) rl_method = SoftActorCritic( env, replay_buffer, feat_ext, play_interval=500, learning_rate=args.lr_rl, buffer_sample_size=args.replay_buffer_sample_size, ) if args.rl_method == "discrete_SAC": if not isinstance(env.action_space, gym.spaces.Discrete): print( "discrete SAC requires a discrete action space environmnet to work." ) exit() replay_buffer = ReplayBuffer(args.replay_buffer_size) rl_method = QSAC( env, replay_buffer, feat_ext, args.replay_buffer_sample_size, learning_rate=args.lr_rl, entropy_tuning=True, entropy_target=0.3, play_interval=args.play_interval, ) print("RL method initialized.") print(rl_method.policy) if args.policy_path is not None: rl_method.policy.load(args.policy_path) experiment_logger.log_header("Details of the RL method :") experiment_logger.log_info(rl_method.__dict__) # initialize IRL method # CHANGE HERE trajectory_path = args.exp_trajectory_path if args.scale_svf is None: scale = False if args.scale_svf: scale = args.scale_svf irl_method = DeepMaxEnt( trajectory_path, rlmethod=rl_method, env=env, iterations=args.irl_iterations, on_server=args.on_server, l1regularizer=args.regularizer, learning_rate=args.lr_irl, seed=args.seed, graft=False, scale_svf=scale, hidden_dims=args.reward_net_hidden_dims, clipping_value=args.clipping_value, enumerate_all=True, save_folder=parent_dir, rl_max_ep_len=args.rl_ep_length, rl_episodes=args.rl_episodes, ) print("IRL method intialized.") print(irl_method.reward) experiment_logger.log_header("Details of the IRL method :") experiment_logger.log_info(irl_method.__dict__) smoothing_flag = False if args.svf_smoothing: smoothing_flag = True irl_method.train(smoothing=smoothing_flag) if not args.dont_save: pass
def plot_reward_across_policy_models(foldername, expert=None, feature_extractor=None, seed_list=[], iterations_per_model=50, compare_expert=True): #given a folder of policy networks, the function will go through them one by one and #create a plot of the rewards obtained by each of the policy networks and compare them #to that of an expert (if provided) color_list = ['r', 'g', 'b', 'c', 'm', 'y', 'k'] counter = 0 reward_across_seeds = [] xaxis = None for seed in seed_list: env = GridWorld(display=False, is_onehot=False, is_random=True, rows=10, cols=10, seed=seed, obstacles=[ np.asarray([5, 1]), np.array([5, 9]), np.asarray([4, 1]), np.array([6, 9]), np.asarray([3, 1]), np.array([7, 9]) ], goal_state=np.asarray([1, 5])) max_ep_length = 20 rl_method = ActorCritic(env, feat_extractor=feature_extractor, gamma=0.99, max_ep_length=max_ep_length, log_interval=50) model_names = glob.glob(os.path.join(foldername, '*.pt')) xaxis = np.arange(len(model_names)) reward_exp = get_rewards_for_model(expert, env=env, feature_extractor=feature_extractor, rl_method=rl_method, max_ep_length=max_ep_length, iterations=iterations_per_model) reward_across_models = [] reward_expert = [] for policy_file in sorted(model_names, key=numericalSort): print('asdfasfsa', policy_file) reward_per_model = get_rewards_for_model( policy_file, env=env, feature_extractor=feature_extractor, rl_method=rl_method, max_ep_length=max_ep_length, iterations=iterations_per_model) print('Average reward for the model:', reward_per_model) reward_across_models.append(reward_per_model) reward_expert.append(reward_exp) reward_across_seeds.append(reward_across_models) np_reward_across_seeds = np.array(reward_across_seeds) print(np_reward_across_seeds.shape) means_rewards = np.mean(np_reward_across_seeds, axis=0) print("the mean rewards :", means_rewards) print("The mean across all runs and seeds : ", np.mean(means_rewards)) std_rewards = np.std(np_reward_across_seeds, axis=0) print('the std :', std_rewards) plt.xlabel('IRL iteration no.') plt.ylabel('Reward obtained') plt.plot(xaxis, means_rewards, color=color_list[counter], label='IRL trained agent') plt.fill_between(xaxis, means_rewards - std_rewards, means_rewards + std_rewards, alpha=0.5, facecolor=color_list[counter]) plt.plot(reward_expert, color='k', label='Expert agent') plt.legend() plt.draw() plt.pause(0.001) plt.show() return reward_across_models
def main(): #####for the logger ts = time.time() st = datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") ################### args = parser.parse_args() seed_all(args.seed) if args.on_server: matplotlib.use("Agg") # pygame without monitor os.environ["SDL_VIDEODRIVER"] = "dummy" from matplotlib import pyplot as plt mp.set_start_method("spawn") from rlmethods.b_actor_critic import ActorCritic from rlmethods.soft_ac import SoftActorCritic, QSoftActorCritic from rlmethods.rlutils import ReplayBuffer from envs.gridworld_drone import GridWorldDrone from featureExtractor.drone_feature_extractor import ( DroneFeatureSAM1, DroneFeatureOccup, DroneFeatureRisk, DroneFeatureRisk_v2, VasquezF1, VasquezF2, VasquezF3, Fahad, GoalConditionedFahad, ) from featureExtractor.gridworld_featureExtractor import ( FrontBackSide, LocalGlobal, OneHot, SocialNav, FrontBackSideSimple, ) from featureExtractor.drone_feature_extractor import ( DroneFeatureRisk_speed, DroneFeatureRisk_speedv2, ) from featureExtractor.drone_feature_extractor import VasquezF1 save_folder = None if not args.dont_save and not args.play: if not args.save_folder: print("Provide save folder.") exit() policy_net_dims = "-policy_net-" for dim in args.policy_net_hidden_dims: policy_net_dims += str(dim) policy_net_dims += "-" reward_net_dims = "-reward_net-" for dim in args.reward_net_hidden_dims: reward_net_dims += str(dim) reward_net_dims += "-" save_folder = ( "./results/" + args.save_folder + st + args.feat_extractor + "-seed-" + str(args.seed) + policy_net_dims + reward_net_dims + "-total-ep-" + str(args.total_episodes) + "-max-ep-len-" + str(args.max_ep_length) ) experiment_logger = Logger(save_folder, "experiment_info.txt") experiment_logger.log_header("Arguments for the experiment :") repo = git.Repo(search_parent_directories=True) experiment_logger.log_info({'From branch : ' : repo.active_branch.name}) experiment_logger.log_info({'Commit number : ' : repo.head.object.hexsha}) experiment_logger.log_info(vars(args)) window_size = 9 step_size = 2 agent_width = 10 obs_width = 10 grid_size = 10 feat_ext = None # initialize the feature extractor to be used if args.feat_extractor == "Onehot": feat_ext = OneHot(grid_rows=10, grid_cols=10) if args.feat_extractor == "SocialNav": feat_ext = SocialNav(fieldList=["agent_state", "goal_state"]) if args.feat_extractor == "FrontBackSideSimple": feat_ext = FrontBackSideSimple( thresh1=1, thresh2=2, thresh3=3, thresh4=4, step_size=step_size, agent_width=agent_width, obs_width=obs_width, ) if args.feat_extractor == "LocalGlobal": feat_ext = LocalGlobal( window_size=11, grid_size=grid_size, agent_width=agent_width, obs_width=obs_width, step_size=step_size, ) if args.feat_extractor == "DroneFeatureSAM1": feat_ext = DroneFeatureSAM1( agent_width=agent_width, obs_width=obs_width, step_size=step_size, grid_size=grid_size, thresh1=15, thresh2=30, ) if args.feat_extractor == "DroneFeatureOccup": feat_ext = DroneFeatureOccup( agent_width=agent_width, obs_width=obs_width, step_size=step_size, grid_size=grid_size, window_size=window_size, ) if args.feat_extractor == "DroneFeatureRisk": feat_ext = DroneFeatureRisk( agent_width=agent_width, obs_width=obs_width, step_size=step_size, grid_size=grid_size, show_agent_persp=False, thresh1=15, thresh2=30, ) if args.feat_extractor == "DroneFeatureRisk_v2": feat_ext = DroneFeatureRisk_v2( agent_width=agent_width, obs_width=obs_width, step_size=step_size, grid_size=grid_size, show_agent_persp=False, thresh1=15, thresh2=30, ) if args.feat_extractor == "DroneFeatureRisk_speed": feat_ext = DroneFeatureRisk_speed( agent_width=agent_width, obs_width=obs_width, step_size=step_size, grid_size=grid_size, show_agent_persp=False, return_tensor=False, thresh1=10, thresh2=15, ) if args.feat_extractor == "DroneFeatureRisk_speedv2": feat_ext = DroneFeatureRisk_speedv2( agent_width=agent_width, obs_width=obs_width, step_size=step_size, grid_size=grid_size, show_agent_persp=False, return_tensor=False, thresh1=18, thresh2=30, ) if args.feat_extractor == "VasquezF1": feat_ext = VasquezF1(agent_width * 6, 0.5, 1.0) if args.feat_extractor == "VasquezF2": feat_ext = VasquezF1(agent_width * 6, 0.5, 1.0) if args.feat_extractor == "VasquezF3": feat_ext = VasquezF3(agent_width) if args.feat_extractor == "Fahad": feat_ext = Fahad(36, 60, 0.5, 1.0) if args.feat_extractor == "GoalConditionedFahad": feat_ext = GoalConditionedFahad(36, 60, 0.5, 1.0) if feat_ext is None: print("Please enter proper feature extractor!") exit() # log feature extractor info if not args.dont_save and not args.play: experiment_logger.log_header("Parameters of the feature extractor :") experiment_logger.log_info(feat_ext.__dict__) # initialize the environment if args.replace_subject: replace_subject = True else: replace_subject = False env = GridWorldDrone( display=args.render, is_onehot=False, seed=args.seed, obstacles=None, show_trail=False, is_random=True, annotation_file=args.annotation_file, subject=args.subject, tick_speed=60, obs_width=10, step_size=step_size, agent_width=agent_width, replace_subject=replace_subject, segment_size=args.segment_size, external_control=True, step_reward=0.001, show_comparison=True, consider_heading=True, show_orientation=True, # rows=200, cols=200, width=grid_size) rows=576, cols=720, width=grid_size, ) # env = gym.make('Acrobot-v1') # log environment info if not args.dont_save and not args.play: experiment_logger.log_header("Environment details :") experiment_logger.log_info(env.__dict__) # initialize RL if args.rl_method == "ActorCritic": model = ActorCritic( env, feat_extractor=feat_ext, gamma=1, log_interval=100, max_episode_length=args.max_ep_length, hidden_dims=args.policy_net_hidden_dims, save_folder=save_folder, lr=args.lr, entropy_coeff=args.entropy_coeff, max_episodes=args.total_episodes, ) if args.rl_method == "SAC": replay_buffer = ReplayBuffer(args.replay_buffer_size) model = SoftActorCritic( env, replay_buffer, feat_ext, buffer_sample_size=args.replay_buffer_sample_size, entropy_tuning=True, play_interval=args.play_interval, entropy_target=args.entropy_target, gamma=args.gamma, learning_rate=args.lr, ) if args.rl_method == "discrete_QSAC": replay_buffer = ReplayBuffer(args.replay_buffer_size) model = QSoftActorCritic( env, replay_buffer, feat_ext, buffer_sample_size=args.replay_buffer_sample_size, entropy_tuning=True, play_interval=args.play_interval, entropy_target=args.entropy_target, gamma=args.gamma, learning_rate=args.lr, ) # log RL info if not args.dont_save and not args.play: experiment_logger.log_header("Details of the RL method :") experiment_logger.log_info(model.__dict__) if args.policy_path is not None: from debugtools import numericalSort policy_file_list = [] reward_across_models = [] # print(args.policy_path) if os.path.isfile(args.policy_path): policy_file_list.append(args.policy_path) if os.path.isdir(args.policy_path): policy_names = glob.glob(os.path.join(args.policy_path, "*.pt")) policy_file_list = sorted(policy_names, key=numericalSort) xaxis = np.arange(len(policy_file_list)) if not args.play and not args.play_user: # no playing of any kind, so training if args.reward_path is None: if args.policy_path: model.policy.load(args.policy_path) if args.rl_method == "SAC" or args.rl_method == "discrete_QSAC": model.train(args.total_episodes, args.max_ep_length) else: model.train() else: from irlmethods.deep_maxent import RewardNet state_size = feat_ext.extract_features(env.reset()).shape[0] reward_net = RewardNet(state_size, args.reward_net_hidden_dims) reward_net.load(args.reward_path) print(next(reward_net.parameters()).is_cuda) model.train(reward_net=reward_net) if not args.dont_save: model.policy.save(save_folder + "/policy-models/") if args.play: # env.tickSpeed = 15 from debugtools import compile_results xaxis = [] counter = 1 plt.figure(0) avg_reward_list = [] frac_good_run_list = [] print(policy_file_list) for policy_file in policy_file_list: print("Playing for policy :", policy_file) model.policy.load(policy_file) policy_folder = policy_file.strip().split("/")[0:-2] save_folder = "" for p in policy_folder: save_folder = save_folder + p + "/" print("The final save folder ", save_folder) # env.tickSpeed = 10 assert args.policy_path is not None, "pass a policy to play from!" if args.exp_trajectory_path is not None: from irlmethods.irlUtils import calculate_expert_svf expert_svf = calculate_expert_svf( args.exp_trajectory_path, max_time_steps=args.max_ep_length, feature_extractor=feat_ext, gamma=1, ) # reward_across_models.append(model.generate_trajectory(args.num_trajs, args.render)) if args.exp_trajectory_path is None: if args.dont_save: rewards, state_info, sub_info = model.generate_trajectory( args.num_trajs, args.render ) else: rewards, state_info, sub_info = model.generate_trajectory( args.num_trajs, args.render, store_raw=args.store_raw_states, path=save_folder + "/agent_generated_trajectories/", ) else: if args.dont_save: rewards, state_info, sub_info = model.generate_trajectory( args.num_trajs, args.render, expert_svf=expert_svf ) else: rewards, state_info, sub_info = model.generate_trajectory( args.num_trajs, args.render, path=save_folder + "/agent_generated_trajectories/", expert_svf=expert_svf, ) avg_reward, good_run_frac = compile_results( rewards, state_info, sub_info ) avg_reward_list.append(avg_reward) frac_good_run_list.append(good_run_frac) plt.plot(avg_reward_list, c="r") plt.plot(frac_good_run_list, c="g") plt.draw() plt.show() if args.play_user: env.tickSpeed = 200 model.generate_trajectory_user( args.num_trajs, args.render, path="./user_generated_trajectories/" )
def main(): args = parser.parse_args() if args.on_server: # matplotlib without monitor matplotlib.use('Agg') # pygame without monitor os.environ['SDL_VIDEODRIVER'] = 'dummy' from rlmethods.rlutils import LossBasedTermination from rlmethods.b_actor_critic import ActorCritic from irlmethods.deep_maxent import DeepMaxEnt import irlmethods.irlUtils as irlUtils from featureExtractor.gridworld_featureExtractor import OneHot, LocalGlobal # initialize the environment #**set is_onehot to false goal_state = np.asarray([5, 5]) env = GridWorld(display=args.render, obstacles=[np.asarray([3, 7])], goal_state=goal_state, step_wrapper=utils.step_wrapper, seed=3, reset_wrapper=utils.reset_wrapper, is_onehot=False) #initialize feature extractor feat_ext = OneHot(grid_rows=10, grid_cols=10) #feat_ext = LocalGlobal(window_size=3, # fieldList = ['agent_state','goal_state','obstacles']) #initialize loss based termination # intialize RL method #pass the appropriate feature extractor rlMethod = ActorCritic(env, gamma=0.99, log_interval=args.rl_log_intervals, max_episodes=args.rl_episodes, max_ep_length=args.rl_ep_length, termination=None, feat_extractor=feat_ext) print("RL method initialized.") if args.policy_path is not None: rlMethod.policy.load(args.policy_path) # initialize IRL method trajectory_path = './trajs/ac_gridworld_locglob_3/' irlMethod = DeepMaxEnt(trajectory_path, rlmethod=rlMethod, env=env, iterations=args.irl_iterations, log_intervals=5, on_server=args.on_server, plot_save_folder='./plots/') print("IRL method intialized.") rewardNetwork = irlMethod.train() if not args.dont_save: pass
def main(): args = parser.parse_args() utils.seed_all(args.seed) if args.on_server: # matplotlib without monitor matplotlib.use('Agg') # pygame without monitor os.environ['SDL_VIDEODRIVER'] = 'dummy' from matplotlib import pyplot as plt save_folder = None if not args.dont_save: save_folder = './results/'+ args.save_folder experiment_logger = Logger(save_folder,'experiment_info.txt') experiment_logger.log_header('Arguments for the experiment :') experiment_logger.log_info(vars(args)) mp.set_start_method('spawn') if args.render: from envs.gridworld import GridWorld else: from envs.gridworld_clockless import GridWorldClockless as GridWorld if args.feat_extractor=='MCFeatures': feat_ext = MCFeatures(args.state_discretization[0], args.state_discretization[1]) elif args.feat_extractor=='MCFeaturesOnehot': feat_ext = MCFeaturesOnehot(args.state_discretization[0], args.state_discretization[1]) else: print('Enter proper feature extractor value.') exit() if not args.dont_save: experiment_logger.log_header('Parameters of the feature extractor :') experiment_logger.log_info(feat_ext.__dict__) ''' np.asarray([2,2]),np.asarray([7,4]),np.asarray([3,5]), np.asarray([5,2]),np.asarray([8,3]),np.asarray([7,5]), np.asarray([3,3]),np.asarray([3,7]),np.asarray([5,7]) env = GridWorld(display=args.render, is_onehot= False,is_random=True, rows=100, agent_width=agent_width,step_size=step_size, obs_width=obs_width,width=grid_size, cols=100, seed=7, buffer_from_obs=0, obstacles=3, goal_state=np.asarray([5,5])) ''' env = gym.make('MountainCar-v0') env = env.unwrapped if not args.dont_save: experiment_logger.log_header('Environment details :') experiment_logger.log_info(env.__dict__) model = ActorCritic(env, feat_extractor=feat_ext, gamma=0.99, plot_loss=False, log_interval=10, max_ep_length=300, hidden_dims=args.policy_net_hidden_dims, max_episodes=30, save_folder=save_folder) if not args.dont_save: experiment_logger.log_header('Details of the RL method :') experiment_logger.log_info(model.__dict__) #pdb.set_trace() if args.policy_path is not None: policy_file_list = [] reward_across_models = [] if os.path.isfile(args.policy_path): policy_file_list.append(args.policy_path) if os.path.isdir(args.policy_path): policy_names = glob.glob(os.path.join(args.policy_path, '*.pt')) policy_file_list = sorted(policy_names, key=numericalSort) xaxis = np.arange(len(policy_file_list)) if not args.play and not args.play_user: if args.reward_path is None: model.train_mp(n_jobs=4) else: from irlmethods.deep_maxent import RewardNet state_size = feat_ext.state_rep_size reward_net = RewardNet(state_size, args.policy_net_hidden_dims) reward_net.load(args.reward_path) print(next(reward_net.parameters()).is_cuda) model.train_mp(reward_net = reward_net,n_jobs = 4) if not args.dont_save: model.policy.save(save_folder+'/policy/') if args.play: xaxis = [] counter = 1 print(policy_file_list) for policy_file in policy_file_list: model.policy.load(policy_file) env.tickSpeed = 15 assert args.policy_path is not None, 'pass a policy to play from!' reward_across_models.append(model.generate_trajectory(args.num_trajs, args.render)) #plotting the 2d list xaxis.append(counter) counter += 1 reward_across_models_np = np.array(reward_across_models) mean_rewards = np.mean(reward_across_models_np, axis=1) std_rewards = np.std(reward_across_models_np, axis=1) plt.plot(xaxis,mean_rewards,color = 'r',label='IRL trained agent') plt.fill_between(xaxis , mean_rewards-std_rewards , mean_rewards+std_rewards, alpha = 0.5, facecolor = 'r') plt.draw() plt.pause(0.001) ''' print('RAM usage :') display_memory_usage(process.memory_info().rss) print('GPU usage :') display_memory_usage(torch.cuda.memory_allocated()) torch.cuda.empty_cache() display_memory_usage(torch.cuda.memory_allocated()) ''' #plt.show() plt.show() if args.play_user: env.tickSpeed = 200 model.generate_trajectory_user(args.num_trajs, './trajs/ac_gridworld_user/')
def main(): args = parser.parse_args() mp.set_start_method('spawn') from envs.gridworld_drone import GridWorldDrone agent_width = 10 step_size = 2 obs_width = 10 grid_size = 10 if args.feat_extractor == 'Onehot': feat_ext = OneHot(grid_rows=10, grid_cols=10) if args.feat_extractor == 'SocialNav': feat_ext = SocialNav(fieldList=['agent_state', 'goal_state']) if args.feat_extractor == 'FrontBackSideSimple': feat_ext = FrontBackSideSimple( thresh1=1, thresh2=2, thresh3=3, thresh4=4, step_size=step_size, agent_width=agent_width, obs_width=obs_width, fieldList=['agent_state', 'goal_state', 'obstacles']) if args.feat_extractor == 'LocalGlobal': feat_ext = LocalGlobal( window_size=3, grid_size=grid_size, agent_width=agent_width, obs_width=obs_width, step_size=step_size, fieldList=['agent_state', 'goal_state', 'obstacles']) #featExtract = OneHot(grid_rows=10,grid_cols=10) #featExtract = FrontBackSideSimple(thresh1 = 1,fieldList = ['agent_state','goal_state','obstacles']) #featExtract = SocialNav(fieldList = ['agent_state','goal_state']) ''' np.asarray([2,2]),np.asarray([7,4]),np.asarray([3,5]), np.asarray([5,2]),np.asarray([8,3]),np.asarray([7,5]), np.asarray([3,3]),np.asarray([3,7]),np.asarray([5,7]) env = GridWorld(display=args.render, is_onehot= False,is_random=True, rows=10, agent_width=agent_width,step_size=step_size, obs_width=obs_width,width=grid_size, cols=10, seed = 7, obstacles = '../envs/map3.jpg', goal_state = np.asarray([5,5])) ''' env = GridWorldDrone(display=args.render, is_onehot=False, seed=999, obstacles=None, show_trail=False, is_random=False, annotation_file=args.annotation_file, subject=None, tick_speed=90, obs_width=10, step_size=step_size, agent_width=agent_width, show_comparison=True, rows=576, cols=720, width=grid_size) model = ActorCritic(env, feat_extractor=featExtract, gamma=0.99, log_interval=50, max_ep_length=500, max_episodes=2000) if args.policy_path is not None: model.policy.load(args.policy_path) if not args.play and not args.play_user: if args.reward_path is None: model.train_mp(n_jobs=4) else: from irlmethods.deep_maxent import RewardNet state_size = featExtract.extract_features(env.reset()).shape[0] reward_net = RewardNet(state_size) reward_net.load(args.reward_path) print(next(reward_net.parameters()).is_cuda) model.train_mp(reward_net=reward_net, n_jobs=4) if not args.dont_save: model.policy.save('./saved-models/') if args.play: #env.tickSpeed = 15 assert args.policy_path is not None, 'pass a policy to play from!' model.generate_trajectory( args.num_trajs, './trajs/ac_loc_glob_rectified_win_3_static_map3/') if args.play_user: env.tickSpeed = 200 model.generate_trajectory_user(args.num_trajs, './trajs/ac_gridworld_user/')
def generate_agent_grid_visitation_map(policy_fname_list, feature_extractor=None, store=False): #given the policy file name list and feature extractor creates a heatmap of the #agent on the gridworld based on the trajectories in the list #if store=True, the figure is stored in the form of a pickle #list containing the points of trajectories of all the policies trajectory_point_master_list = [] traj_to_plot = 2 env = GridWorld(display=False, is_onehot=False, is_random=False, rows=10, cols=10, seed=3, obstacles=[np.asarray([5, 5])], goal_state=np.asarray([1, 5])) max_ep_length = 15 run_iterations = 50 rl_method = ActorCritic(env, feat_extractor=feature_extractor, gamma=0.99, max_ep_length=max_ep_length, log_interval=50) labels = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'] counter = 0 for name in policy_fname_list: counter += 1 if counter == traj_to_plot: policy_name_to_plot = name #ready the policy rl_method.policy.load(name) trajectory_point_policy = [] env = GridWorld(display=False, is_onehot=False, is_random=False, rows=10, cols=10, seed=7, obstacles=[np.asarray([5, 5])], goal_state=np.asarray([1, 5])) heat_map = np.zeros((env.rows, env.cols)) for i in range(run_iterations): trajectory_point_run = [] state = env.reset() heat_map[state['agent_state'][0], state['agent_state'][1]] += 1 trajectory_point_run.append( (state['agent_state'][0] * env.cellWidth, state['agent_state'][1] * env.cellWidth)) state = feature_extractor.extract_features(state) for t in range(max_ep_length): action = rl_method.select_action(state) state, reward, done, _ = env.step(action) heat_map[state['agent_state'][0], state['agent_state'][1]] += 1 trajectory_point_run.append( (state['agent_state'][0] * env.cellWidth, state['agent_state'][1] * env.cellWidth)) state = feature_extractor.extract_features(state) trajectory_point_policy.append(trajectory_point_run) trajectory_point_master_list.append(trajectory_point_policy) fig, ax = plt.subplots() im = ax.imshow(heat_map, vmin=0, vmax=40) ax.set_xticks(np.arange(10)) ax.set_yticks(np.arange(10)) ax.set_xticklabels(labels) ax.set_yticklabels(labels) ax.set_xlabel('Columns of the gridworld', fontsize='large') ax.set_ylabel('Rows of the gridworld', fontsize='large') for i in range(len(labels)): for j in range(len(labels)): text = ax.text(j, i, heat_map[i, j], ha="center", va="bottom", color="black") #arrow = ax.arrow(j,i,.1,.1,shape='full',head_width= .2) #arrow = ax.annotate("",xy = (j,i) , arrowprops = arrow) pass ax.set_title("Grid location visitation frequency for a unbiased agent") #plt.colorbar() #plt.clim(0,70) plt.draw() if store: pickle_filename = 'FigureObject' + str(counter) + '.fig.pickle' pickle.dump(fig, open(pickle_filename, 'wb')) plt.pause(.001) #annotate_trajectory(policy_name_to_plot, env, rl_method, # max_ep_length, ax, feature_extractor=feature_extractor) plt.show()
def main(): """Runs experiment""" args = parser.parse_args() utils.seed_all(args.seed) ts = time.time() st = datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%d_%H:%M:%S") to_save = pathlib.Path(args.save_dir) dir_name = args.save_folder + "_" + st to_save = to_save / dir_name to_save = str(to_save.resolve()) log_file = "Experiment_info.txt" experiment_logger = Logger(to_save, log_file) experiment_logger.log_header("Arguments for the experiment :") experiment_logger.log_info(vars(args)) feat_ext = fe_utils.load_feature_extractor(args.feat_extractor, obs_width=args.pedestrian_width, agent_width=args.pedestrian_width) experiment_logger.log_header("Parameters of the feature extractor :") experiment_logger.log_info(feat_ext.__dict__) env = GridWorld( display=args.render, is_random=False, rows=576, cols=720, agent_width=args.pedestrian_width, step_size=2, obs_width=args.pedestrian_width, width=10, subject=args.subject, annotation_file=args.annotation_file, goal_state=None, step_wrapper=utils.step_wrapper, seed=args.seed, replace_subject=args.replace_subject, segment_size=args.segment_size, external_control=True, continuous_action=False, reset_wrapper=utils.reset_wrapper, consider_heading=True, is_onehot=False, show_orientation=True, show_comparison=True, show_trail=True, ) experiment_logger.log_header("Environment details :") experiment_logger.log_info(env.__dict__) if args.rl_method == "ActorCritic": rl_method = ActorCritic( env, feat_extractor=feat_ext, gamma=1, log_interval=args.rl_log_intervals, max_episode_length=args.rl_ep_length, hidden_dims=args.policy_net_hidden_dims, save_folder=to_save, lr=args.lr_rl, max_episodes=args.rl_episodes, ) if args.rl_method == "SAC": if not env.continuous_action: print("The action space needs to be continuous for SAC to work.") exit() replay_buffer = ReplayBuffer(args.replay_buffer_size) rl_method = SoftActorCritic( env, replay_buffer, feat_ext, play_interval=500, learning_rate=args.lr_rl, buffer_sample_size=args.replay_buffer_sample_size, ) if args.rl_method == "discrete_QSAC": if not isinstance(env.action_space, gym.spaces.Discrete): print("discrete SAC requires a discrete action space to work.") exit() replay_buffer = ReplayBuffer(args.replay_buffer_size) rl_method = QSAC( env, replay_buffer, feat_ext, args.replay_buffer_sample_size, learning_rate=args.lr_rl, entropy_tuning=True, entropy_target=args.entropy_target, play_interval=args.play_interval, tau=args.tau, gamma=args.gamma, ) if args.rl_method == "discrete_SAC": if not isinstance(env.action_space, gym.spaces.Discrete): print("discrete SAC requires a discrete action space to work.") exit() replay_buffer = ReplayBuffer(args.replay_buffer_size) rl_method = DiscreteSAC( env, replay_buffer, feat_ext, args.replay_buffer_sample_size, learning_rate=args.lr_rl, entropy_tuning=True, entropy_target=args.entropy_target, play_interval=args.play_interval, tau=args.tau, gamma=args.gamma, ) print("RL method initialized.") print(rl_method.policy) if args.policy_path is not None: rl_method.policy.load(args.policy_path) experiment_logger.log_header("Details of the RL method :") experiment_logger.log_info(rl_method.__dict__) expert_trajectories = read_expert_trajectories(args.exp_trajectory_path) irl_method = PerTrajGCL( rl=rl_method, env=env, expert_trajectories=expert_trajectories, learning_rate=args.lr_irl, l2_regularization=args.regularizer, save_folder=to_save, saving_interval=args.saving_interval, ) print("IRL method intialized.") print(irl_method.reward_net) experiment_logger.log_header("Details of the IRL method :") experiment_logger.log_info(irl_method.__dict__) irl_method.pre_train( args.pre_train_iterations, args.num_expert_samples, account_for_terminal_state=args.account_for_terminal_state, gamma=args.gamma, ) rl_method.train( args.pre_train_rl_iterations, args.rl_ep_length, reward_network=irl_method.reward_net, ) # save intermediate RL result rl_method.policy.save(to_save + "/policy") irl_method.train( args.irl_iterations, args.rl_episodes, args.rl_ep_length, args.rl_ep_length, reset_training=args.reset_training, account_for_terminal_state=args.account_for_terminal_state, gamma=args.gamma, stochastic_sampling=args.stochastic_sampling, num_expert_samples=args.num_expert_samples, num_policy_samples=args.num_policy_samples, ) metric_applicator = metric_utils.LTHMP2020() metric_results = metric_utils.collect_trajectories_and_metrics( env, feat_ext, rl_method.policy, len(expert_trajectories), args.rl_ep_length, metric_applicator, disregard_collisions=True, ) pd_metrics = pd.DataFrame(metric_results).T pd_metrics = pd_metrics.applymap(lambda x: x[0]) pd_metrics.to_pickle(to_save + "/metrics.pkl") with open(to_save + "/rl_data.csv", "a") as f: rl_method.data_table.write_csv(f) with open(to_save + "/irl_data.csv", "a") as f: irl_method.data_table.write_csv(f) with open(to_save + "/pre_irl_data.csv", "a") as f: irl_method.pre_data_table.write_csv(f)
width=grid_size) #load the feature extractor feat_ext = DroneFeatureRisk_speed(agent_width=agent_width, obs_width=obs_width, step_size=step_size, grid_size=grid_size, show_agent_persp=False, thresh1=10, thresh2=15) #load the actor critic module model = ActorCritic(env, feat_extractor=feat_ext, gamma=1, log_interval=100, max_episode_length=max_ep_length, hidden_dims=policy_net_hidden_dims, save_folder=None, max_episodes=total_episodes) expert_trajectory_folder = '/home/abhisek/Study/Robotics/deepirl/envs/expert_datasets/university_students/annotation/traj_info/frame_skip_1/students003/DroneFeatureRisk_speed_smooth_state/' policy_folder = '/home/abhisek/Study/Robotics/deepirl/experiments/results/Alienware/Temp' policy_file_list = [] #read the files in the folder if os.path.isdir(policy_folder): policy_names = glob.glob(os.path.join(policy_folder, '*.pt')) policy_file_list = sorted(policy_names, key=numericalSort) else: policy_file_list.append(policy_folder)