def main(load_policy=True): global log_dir model_class = TD3 # works also with SAC and DDPG action_space = 6 gamma = 0.9 memory_limit = 1000000 timesteps = 15000000 discreteAction = 0 rend = False # learning rate env = bioEnv() env = Monitor(env, log_dir, allow_early_resets=True) goal_selection_strategy = 'future' n_actions = env.action_space.shape[-1] action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions)) # Wrap the model model = HER(CustomTD3Policy, env, model_class,n_sampled_goal=4, goal_selection_strategy=goal_selection_strategy, verbose=1,tensorboard_log="../pybullet_logs/bioEnv_TD3", buffer_size=1000000,batch_size= 256, random_exploration=0.3, action_noise=action_noise) if (load_policy): model = HER.load("models/TD3/curriculum/best_model_part_11_10g_TRUE.pkl", env=env, n_sampled_goal=4, goal_selection_strategy=goal_selection_strategy, tensorboard_log="../pybullet_logs/bioEnv_TD3", buffer_size=1000000,batch_size=256,random_exploration=0.3, action_noise=action_noise) model.learn(timesteps,log_interval=100, callback = callback) model.save("policy_TD3_Discr")
def train_policy(num_of_envs, log_relative_path, maximum_episode_length, skip_frame, seed_num, her_config, total_time_steps, validate_every_timesteps, task_name): task = generate_task(task_generator_id=task_name, dense_reward_weights=np.array([100000, 0, 0, 0]), fractional_reward_weight=0) env = CausalWorld(task=task, skip_frame=skip_frame, enable_visualization=False, seed=seed_num, max_episode_length=maximum_episode_length) env = HERGoalEnvWrapper(env) env = CurriculumWrapper( env, intervention_actors=[GoalInterventionActorPolicy()], actives=[(0, 1000000000, 1, 0)]) set_global_seeds(seed_num) checkpoint_callback = CheckpointCallback(save_freq=int( validate_every_timesteps / num_of_envs), save_path=log_relative_path, name_prefix='model') model = HER(MlpPolicy, env, SAC, verbose=1, policy_kwargs=dict(layers=[256, 256, 256]), **her_config, seed=seed_num) model.learn(total_timesteps=total_time_steps, tb_log_name="her_sac", callback=checkpoint_callback) return
def train_policy(num_of_envs, log_relative_path, maximum_episode_length, skip_frame, seed_num, sac_config, total_time_steps, validate_every_timesteps, task_name): def _make_env(rank): def _init(): task = generate_task(task_generator_id=task_name) env = CausalWorld(task=task, skip_frame=skip_frame, enable_visualization=False, seed=seed_num + rank, max_episode_length=maximum_episode_length) env = HERGoalEnvWrapper(env) return env set_global_seeds(seed_num) return _init os.makedirs(log_relative_path) env = SubprocVecEnv([_make_env(rank=i) for i in range(num_of_envs)]) model = HER('MlpPolicy', env, SAC, verbose=1, policy_kwargs=dict(layers=[256, 256, 256]), **sac_config) save_config_file(sac_config, _make_env(0)(), os.path.join(log_relative_path, 'config.json')) for i in range(int(total_time_steps / validate_every_timesteps)): model.learn(total_timesteps=validate_every_timesteps, tb_log_name="sac", reset_num_timesteps=False) model.save(os.path.join(log_relative_path, 'saved_model')) return
def main(argv): numControlledJoints = 6 fixed = False normalize_observations = False gamma = 0.9 batch_size = 16 memory_limit = 1000000 normalize_returns = True timesteps = 1000000 policy_name = "reaching_policy" discreteAction = 0 rend = False kukaenv = kukaReachGymEnvHer(urdfRoot=robot_data.getDataPath(), renders=rend, useIK=0, isDiscrete=discreteAction, numControlledJoints=numControlledJoints, fixedPositionObj=fixed, includeVelObs=True) kukaenv = Monitor(kukaenv, log_dir, allow_early_resets=True) n_actions = kukaenv.action_space.shape[-1] param_noise = None action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions)) model_class = DDPG goal_selection_strategy = 'future' model = HER(CustomPolicy, kukaenv, model_class, n_sampled_goal=4, goal_selection_strategy=goal_selection_strategy, verbose=1, tensorboard_log= "../pybullet_logs/kuka_reach_ddpg/reaching_DDPG_HER_PHASE", buffer_size=1000000, batch_size=64, random_exploration=0.3, action_noise=action_noise) print(colored("-----Timesteps:", "red")) print(colored(timesteps, "red")) print(colored("-----Number Joints Controlled:", "red")) print(colored(numControlledJoints, "red")) print(colored("-----Object Position Fixed:", "red")) print(colored(fixed, "red")) print(colored("-----Policy Name:", "red")) print(colored(policy_name, "red")) print(colored("------", "red")) print(colored("Launch the script with -h for further info", "red")) model.learn(total_timesteps=timesteps, log_interval=100, callback=callback) print("Saving model to kuka.pkl") model.save("../pybullet_logs/kukareach_ddpg_her/" + policy_name) del model # remove to demonstrate saving and loading
def test_long_episode(model_class): """ Check that the model does not break when the replay buffer is still empty after the first rollout (because the episode is not over). """ # n_bits > nb_rollout_steps n_bits = 10 env = BitFlippingEnv(n_bits, continuous=model_class in [DDPG, SAC, TD3], max_steps=n_bits) kwargs = {} if model_class == DDPG: kwargs['nb_rollout_steps'] = 9 # < n_bits elif model_class in [DQN, SAC, TD3]: kwargs['batch_size'] = 8 # < n_bits kwargs['learning_starts'] = 0 model = HER('MlpPolicy', env, model_class, n_sampled_goal=4, goal_selection_strategy='future', verbose=0, **kwargs) model.learn(100)
def main( training_env: PSMCartesianHERDDPGEnv, eval_env: PSMCartesianHERDDPGEnv = None, log_dir='./.logs/results' ): os.makedirs(log_dir, exist_ok=True) # training_env = Monitor(training_env, log_dir) n_actions = training_env.action_space.shape[0] noise_std = 0.2 # Currently using OU noise action_noise = OrnsteinUhlenbeckActionNoise( mean=np.zeros(n_actions), sigma=noise_std * np.ones(n_actions) ) model_class = DDPG # works also with SAC, DDPG and TD3 rl_model_kwargs = { 'actor_lr': 1e-3, 'critic_lr': 1e-3, 'action_noise': action_noise, 'nb_train_steps': 300, 'nb_rollout_steps': 100, 'gamma': 0.95, 'observation_range': (-1.5, 1.5), 'random_exploration': 0.05, 'normalize_observations': True, 'critic_l2_reg': 0.01 } # Available strategies (cf paper): future, final, episode, random model = HER( 'MlpPolicy', training_env, model_class, verbose=1, n_sampled_goal=4, goal_selection_strategy='future', buffer_size=int(1e5), batch_size=128, tensorboard_log="./ddpg_dvrk_tensorboard/", **rl_model_kwargs ) # Reset the model training_env.reset() # Create callbacks checkpoint_callback = CheckpointCallback( save_freq=100000, save_path="./ddpg_dvrk_tensorboard/" ) # save_path="./.model/model_checkpoint/") #save_freq=100000 # eval_callback = EvalCallback(training_env, best_model_save_path='./ddpg_dvrk_tensorboard/best_model', # log_path=log_dir, eval_freq=500) callback = CallbackList([checkpoint_callback]) # , eval_callback]) # Train the model model.learn(4000000, log_interval=100, callback=callback) model.save("./her_robot_env")
def test_model_manipulation(model_class, goal_selection_strategy): env = BitFlippingEnv(N_BITS, continuous=model_class in [DDPG, SAC], max_steps=N_BITS) env = DummyVecEnv([lambda: env]) model = HER('MlpPolicy', env, model_class, n_sampled_goal=3, goal_selection_strategy=goal_selection_strategy, verbose=0) model.learn(1000) model_predict(model, env, n_steps=100, additional_check=None) model.save('./test_her') del model # NOTE: HER does not support VecEnvWrapper yet with pytest.raises(AssertionError): model = HER.load('./test_her', env=VecNormalize(env)) model = HER.load('./test_her') # Check that the model raises an error when the env # is not wrapped (or no env passed to the model) with pytest.raises(ValueError): model.predict(env.reset()) env_ = BitFlippingEnv(N_BITS, continuous=model_class in [DDPG, SAC], max_steps=N_BITS) env_ = HERGoalEnvWrapper(env_) model_predict(model, env_, n_steps=100, additional_check=None) model.set_env(env) model.learn(1000) model_predict(model, env_, n_steps=100, additional_check=None) assert model.n_sampled_goal == 3 del model env = BitFlippingEnv(N_BITS, continuous=model_class in [DDPG, SAC], max_steps=N_BITS) model = HER.load('./test_her', env=env) model.learn(1000) model_predict(model, env_, n_steps=100, additional_check=None) assert model.n_sampled_goal == 3 if os.path.isfile('./test_her.pkl'): os.remove('./test_her.pkl')
def main(load_policy=False): global log_dir model_class = TD3 # works also with SAC and DDPG action_space = 7 normalize_observations = False gamma = 0.9 memory_limit = 1000000 normalize_returns = True timesteps = 8000000 rend = False obj_pose_rnd_std = 0 env = pandaPushGymGoalEnv(renders=rend, use_IK=0, numControlledJoints=action_space, obj_pose_rnd_std=obj_pose_rnd_std, includeVelObs=True) env = Monitor(env, log_dir, allow_early_resets=True) goal_selection_strategy = 'future' n_actions = env.action_space.shape[-1] action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions)) # Wrap the model model = HER( CustomTD3Policy, env, model_class, n_sampled_goal=4, goal_selection_strategy=goal_selection_strategy, verbose=1, tensorboard_log= "../pybullet_logs/panda_push_TD3/stable_baselines/TD3_phase1_target_fixed", buffer_size=1000000, batch_size=256, random_exploration=0.3, action_noise=action_noise) if (load_policy): model = HER.load( "../policies/USEFUL_POLICIES/PUSHING_TD3+HER_FIXED_POSITIONbest_model.pkl", env=env, n_sampled_goal=4, goal_selection_strategy=goal_selection_strategy, tensorboard_log= "../pybullet_logs/panda_push_TD3/stable_baselines/TD3_phase1_target_fixed", buffer_size=1000000, batch_size=256, random_exploration=0.3, action_noise=action_noise) model.learn(timesteps, log_interval=100, callback=callback) print("Saving Policy PHASE_1") model.save("../policies/TD3_phase1_target_fixed")
def test_her(model_class, goal_selection_strategy, discrete_obs_space): env = BitFlippingEnv(N_BITS, continuous=model_class in [DDPG, SAC, TD3], max_steps=N_BITS, discrete_obs_space=discrete_obs_space) # Take random actions 10% of the time kwargs = {'random_exploration': 0.1} if model_class in [DDPG, SAC, TD3] else {} model = HER('MlpPolicy', env, model_class, n_sampled_goal=4, goal_selection_strategy=goal_selection_strategy, verbose=0, **kwargs) model.learn(1000)
def main(): model_class = DDPG # works also with SAC and DDPG # -j action_space = 7 # -p fixed = True # -o normalize_observations = False # -g gamma = 0.9 # -b #batch_size = 16 # -m memory_limit = 1000000 # -r normalize_returns = True # -t timesteps = 1000000 policy_name = "pushing_policy" discreteAction = 0 rend = False env = pandaPushGymEnvHERRand(urdfRoot=robot_data.getDataPath(), renders=rend, useIK=0, isDiscrete=discreteAction, action_space=action_space, fixedPositionObj=fixed, includeVelObs=True) # Available strategies (cf paper): future, final, episode, random goal_selection_strategy = 'future' # equivalent to GoalSelectionStrategy.FUTURE n_actions = env.action_space.shape[-1] action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions)) # Wrap the model model = HER( CustomPolicy, env, model_class, n_sampled_goal=4, goal_selection_strategy=goal_selection_strategy, verbose=1, tensorboard_log= "../pybullet_logs/panda_push_ddpg/stable_baselines/DDPG+HER_FIXED_DYN_RAND", buffer_size=1000000, batch_size=256, random_exploration=0.3, action_noise=action_noise) # Train the model starting from a previous policy model.learn(timesteps) print("Saving Policy") model.save("../policies/pushing_fixed_HER_Dyn_Rand")
def launchAgent(model_name: str): """ :param model_name: 실행시킬 모델의 종류. HER, DDPG, PPO2 혹은 기타값(DQN)이어야 함 현재는 의도상 PPO2로 세팅할 것 :return: 1000회의 사이클을 돌고 난 이후의 모델 """ import Reinforcement_AI.env.e_enhanced_image_env as image_env from stable_baselines import DQN, HER, DDPG, PPO2 from stable_baselines.common import make_vec_env print("Current Env is " + model_name) if model_name == "HER": env = image_env.DetailedMiniMapEnv() model = HER("CnnPolicy", env=env, model_class=DQN) if model_name == "DDPG": env = image_env.DDPGImageEnv() model = DDPG(policy="CnnPolicy", env=env, normalize_observations=True) if model_name == "PPO2": env = make_vec_env(image_env.DetailedMiniMapEnv, n_envs=1) model = PPO2(policy="CnnPolicy", env=env, verbose=1) else: env = image_env.DetailedMiniMapEnv() model = DQN( "CnnPolicy", # policy env=env, # environment double_q=True, # Double Q enable prioritized_replay=True, # Replay buffer enabled verbose=0 # log print ) for i in range(1000): if i != 0: if model_name == "HER": model = HER.load("detailedmap_HER_" + str(i), env) if model_name == "DDPG": model = DDPG.load("detailedmap_DDPG_" + str(i), env) if model_name == "PPO2": model = PPO2.load("detailedmap_PPO2_" + str(i), env) else: model = DQN.load("detailedmap_DQN_" + str(i), env) # print('model learn start') model.learn(total_timesteps=12500) #FPS가 130이상 넘어갈때의 최소수치 print("this model is : detailedmap_" + model_name + "_" + str(i + 1)) # print('model learn finished') # print('model save start') model.save("detailedmap_" + model_name + "_" + str(i + 1)) del model # print('model save end') return model
def heralgorithm(): goal_selection_strategy = 'future' # equivalent to GoalSelectionStrategy.FUTURE # Wrap the model model = HER('MlpPolicy', env1, DDPG, n_sampled_goal=4, goal_selection_strategy=goal_selection_strategy, verbose=1) # Train the model model.learn(1000) model.save("./her_bit_env")
def train_HER(env_train, model_name, timesteps=50000): start = time.time() n_sampled_goal = 4 goal_selection_strategy = 'future' model = HER('MlpPolicy', env_train, model_class=SAC, verbose=0, n_sampled_goal=n_sampled_goal, goal_selection_strategy=goal_selection_strategy) model.learn(total_timesteps=timesteps) end = time.time() model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}") print('Training time (HER): ', (end - start) / 60, ' minutes') return model
def main(env): n_actions = env.action_space.shape[0] noise_std = 0.2 # Currently using OU noise action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=noise_std * np.ones(n_actions)) model_class = DDPG # works also with SAC, DDPG and TD3 rl_model_kwargs = { 'actor_lr': 1e-3, 'critic_lr': 1e-3, 'action_noise': action_noise, 'nb_train_steps': 300, 'nb_rollout_steps': 100, 'gamma': 0.95, 'observation_range': (-1.5, 1.5), 'random_exploration': 0.05, 'normalize_observations': True, 'critic_l2_reg': 0.01 } # Available strategies (cf paper): future, final, episode, random model = HER('MlpPolicy', env, model_class, verbose=1, n_sampled_goal=4, goal_selection_strategy='future', buffer_size=int(1e5), batch_size=128, tensorboard_log="./ddpg_dvrk_tensorboard/", **rl_model_kwargs) # Reset the model env.reset() # Train the model model.learn(4000000, log_interval=100, callback=CheckpointCallback( save_freq=100000, save_path="./ddpg_dvrk_tensorboard/")) model.save("./her_robot_env")
def main(load_policy=False): global log_dir, log_dir_policy if (load_policy): log_dir_policy = '../policies/PUSHING_TD3+HER_FIXED_POSITION_DYN_RAND_FROM_FIXED_PHYSICS' model_class = TD3 # works also with SAC and DDPG action_space = 7 fixed = True normalize_observations = False gamma = 0.9 memory_limit = 1000000 normalize_returns = True timesteps = 1500000 discreteAction = 0 rend = False env = pandaPushGymEnvHERRand(urdfRoot=robot_data.getDataPath(), renders=rend, useIK=0, isDiscrete=discreteAction, action_space = action_space, fixedPositionObj = fixed, includeVelObs = True) env = Monitor(env, log_dir, allow_early_resets=True) # Available strategies (cf paper): future, final, episode, random goal_selection_strategy = 'future' # equivalent to GoalSelectionStrategy.FUTURE n_actions = env.action_space.shape[-1] action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions)) # Wrap the model model = HER(CustomPolicy, env, model_class, n_sampled_goal=4, goal_selection_strategy=goal_selection_strategy, verbose=1,tensorboard_log="../pybullet_logs/panda_push_TD3/stable_baselines/TD3+HER_FIXED_DYN_RAND", buffer_size=1000000,batch_size=256, random_exploration=0.3, action_noise=action_noise) if (load_policy): model = HER.load("../policies/USEFUL_POLICIES/PUSHING_TD3+HER_FIXED_POSITIONbest_model.pkl", env=env, n_sampled_goal=4, goal_selection_strategy=goal_selection_strategy, tensorboard_log="../pybullet_logs/panda_push_TD3/stable_baselines/TD3+HER_FIXED_DYN_RAND_FROM_FIXED_PHYSICS", buffer_size=1000000,batch_size=256,random_exploration=0.3, action_noise=action_noise) # Train the model starting from a previous policy model.learn(timesteps, callback = callback ) model.save("../policies/PUSHING_FIXED_TD3_DYN_RAND") print("Finished train1")
def main(load_policy=False): global log_dir model_class = TD3 # works also with SAC and DDPG action_space = 6 fixed = True #0 completely fixed, 1 slightly random radius, 2 big random radius, object_position = 1 normalize_observations = False gamma = 0.9 memory_limit = 1000000 normalize_returns = True timesteps = 5000000 discreteAction = 0 rend = False env = pandaPushGymEnvHER(urdfRoot=robot_data.getDataPath(), renders=rend, useIK=1, isDiscrete=discreteAction, action_space = action_space, fixedPositionObj = fixed, includeVelObs = True, object_position=object_position) env = Monitor(env, log_dir, allow_early_resets=True) goal_selection_strategy = 'future' n_actions = env.action_space.shape[-1] action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions)) # Wrap the model model = HER(CustomTD3Policy, env, model_class, n_sampled_goal=4, goal_selection_strategy=goal_selection_strategy, verbose=1,tensorboard_log="../pybullet_logs/panda_push_TD3/stable_baselines/PUSHING_TD3+HER_FIXED_POSITION_PHASE_1_IK", buffer_size=1000000,batch_size=256, random_exploration=0.3, action_noise=action_noise) if (load_policy): model = HER.load("../policies/USEFUL_POLICIES/PUSHING_TD3+HER_FIXED_POSITIONbest_model.pkl", env=env, n_sampled_goal=4, goal_selection_strategy=goal_selection_strategy, tensorboard_log="../pybullet_logs/panda_push_TD3/stable_baselines/PUSHING_TD3+HER_FIXED_POSITION_PHASE_1_IK", buffer_size=1000000,batch_size=256,random_exploration=0.3, action_noise=action_noise) model.learn(timesteps,log_interval=100, callback = callback) print("Saving Policy PHASE_1") model.save("../policies/PUSHING_TD3+HER_FIXED_POSITION_PHASE_1_IK")
def __init__(self, env: ISettableGoalEnv, verbose=1, rank=0, experiment_name="her-sac"): self._env = env self._dirs = Dirs( experiment_name=f"{type(env).__name__}-{experiment_name}", rank=rank) options = { "env": env, "tensorboard_log": self._dirs.tensorboard, "model_class": SAC, "gamma": 1, "learning_rate": 3e-3 } if os.path.isdir(self._dirs.models) and os.path.isfile( self._dirs.best_model): self._model = HER.load(load_path=self._dirs.best_model, **options) print(f"Loaded model {self._dirs.best_model}") else: self._model = HER(policy="MlpPolicy", verbose=verbose, **options)
os.makedirs(model_path, exist_ok=True) set_global_seeds(0) num_of_active_envs = 1 policy_kwargs = dict(layer=[256, 256]) #env = gym.make("real_robot_challenge_phase_1-v1") env = FlatObservationWrapper( ExamplePushingTrainingEnv(frameskip=20, visualization=False)) train_configs = { "gamma": 0.99, "n_steps": int(120000 / 20), "ent_coef": 0.01, "learning_rate": 0.00025, "vf_coef": 0.5, "max_grad_norm": 0.5, "nminibatches": 40, "noptepochs": 4, } model = HER(MlpPolicy, env, SAC, verbose=1, tensorboard_log=model_path) ckpt_frequency = int(validate_every_timesteps / num_of_active_envs) checkpoint_callback = CheckpointCallback(save_freq=ckpt_frequency, save_path=model_path, name_prefix="model") model.learn(int(total_time_steps), callback=checkpoint_callback) env.close()
def launchAgent(): import Reinforcement_AI.env.d_image_env as image_env from stable_baselines import DQN, HER, DDPG, PPO2 from stable_baselines.common import make_vec_env model_name = "PPO2" if model_name == "HER": model = HER( "CnnPolicy", env=image_env.DetailedMiniMapEnv(), model_class=DQN ) if model_name == "DDPG": model = DDPG( policy="CnnPolicy", env=image_env.DDPGImageEnv(), normalize_observations=True ) if model_name == "PPO2": # env = image_env.DetailedMiniMapEnv() env = make_vec_env(image_env.DetailedMiniMapEnv, n_envs=1) model = PPO2( policy="CnnPolicy", env=env, verbose=1 ) else: model = DQN( "CnnPolicy", # policy env=image_env.DetailedMiniMapEnv(), # environment double_q=True, # Double Q enable prioritized_replay=True, # Replay buffer enabled verbose=0 # log print ) for i in range(1000): if i != 0: if model_name == "HER": model = HER.load("detailedmap_HER_" + str(i)) model.set_env(image_env.DetailedMiniMapEnv()) if model_name == "DDPG": model = DDPG.load("detailedmap_DDPG_" + str(i)) model.set_env(image_env.DDPGImageEnv()) if model_name == "PPO2": # print('set env') # ppo2_env = make_vec_env(image_env.DetailedMiniMapEnv, n_envs=1) # print('get model') model = PPO2.load("detailedmap_PPO2_" + str(i), env) # print('set model env') # model.set_env(ppo2_env) else: model = DQN.load("detailedmap_DQN_" + str(i)) model.set_env(image_env.DetailedMiniMapEnv()) # print('model learn start') model.learn(total_timesteps=3900) # print('model learn finished') # print('model save start') model.save("detailedmap_" + model_name + "_" + str(i+1)) del model
import time env = gym.make("FetchPickAndPlace-v1") goal_selection_strategy = 'future' # equivalent to GoalSelectionStrategy.FUTURE # Wrap the model model = HER('MlpPolicy', env, SAC, n_sampled_goal=4, goal_selection_strategy='future', verbose=1, buffer_size=int(1e6), learning_rate=0.001, gamma=0.95, batch_size=256, ent_coef='auto', random_exploration=0.3, learning_starts=1000, train_freq=1, policy_kwargs=dict(layers=[256, 256, 256]), tensorboard_log="./OpenAI/") # Train the model model.learn(int(8e6)) model.save("./model2") # WARNING: you must pass an env # or wrap your environment with HERGoalEnvWrapper to use the predict method model = HER.load('./model2', env=env)
def train_single(self, env_name="Merging-v0"): """ Directly trains on env_name """ for seed in [201, 202, 203, 204, 205]: print(f"\ntraining with bsize {self.bs}, seed{seed}") self.seed = seed self.experiment_name = f"B{self.bs}R{seed}" print("EXPT NAME: ", self.experiment_dir1, self.experiment_name) self.experiment_dir = os.path.join(self.experiment_dir1, self.experiment_name) self.create_eval_dir() self.model = None env = gym.make(env_name) eval_env = gym.make(env_name) env._set_barrier_size(self.bs) env._set_homotopy_class('right') eval_env._set_barrier_size(self.bs) eval_env._set_homotopy_class('right') if self.model_type == "PPO": if self.is_save: ### DEEPER NETWORK #policy_kwargs = dict(net_arch=[dict(pi=[64, 64, 64, 64], # vf=[64, 64, 64, 64])] # ) #self.PPO = PPO2('MlpPolicy', env, verbose=1, seed=self.seed, learning_rate=1e-3, # policy_kwargs=policy_kwargs) ### DROPOUT #self.PPO = PPO2(MlpGeneralPolicy1, env, verbose=1, seed=self.seed, learning_rate=1e-3) ### REGULAR self.PPO = PPO2('MlpPolicy', env, verbose=1, seed=self.seed, learning_rate=1e-3) else: self.PPO = PPO2('MlpPolicy', env, verbose=1, seed=self.seed, learning_rate=1e-3) self.model = train(self.PPO, eval_env, self.timesteps, self.experiment_dir, self.is_save, self.eval_save_period, self.rets_path, 0) elif self.model_type == "DQN": if self.is_save: self.DQN = DQN( 'MlpPolicy', env, verbose=1, seed=self.seed, prioritized_replay=True, learning_rate=1e-3, tensorboard_log="./Gridworldv1_tensorboard/" + self.experiment_name, full_tensorboard_log=True) else: self.DQN = DQN('MlpPolicy', env, verbose=1, seed=self.seed, prioritized_replay=True, learning_rate=1e-3) self.model = train(self.DQN, eval_env, self.timesteps, self.experiment_dir, self.is_save, self.eval_save_period, self.rets_path, 0) elif self.model_type == "HER": env = HERGoalEnvWrapper(env) eval_env = HERGoalEnvWrapper(eval_env) print("bs: ", env.env.barrier_size) print("hc: ", env.env.homotopy_class) self.HER = HER('MlpPolicy', env, DDPG, n_sampled_goal=4, goal_selection_strategy="future", seed=self.seed, verbose=1) self.model = train(self.HER, eval_env, self.timesteps, self.experiment_dir, self.is_save, self.eval_save_period, self.rets_path, 0)
env_name = 'blueprint_construction' core_dir = '/Users/abhijithneilabraham/Documents/GitHub/multi-agent-emergence-environments/' envs_dir = 'mae_envs/envs' xmls_dir = 'xmls' env, _ = load_env(env_name, core_dir=core_dir, envs_dir=envs_dir, xmls_dir=xmls_dir, return_args_remaining=True) # Available strategies (cf paper): future, final, episode, random goal_selection_strategy = 'future' # equivalent to GoalSelectionStrategy.FUTURE # Wrap the model model = HER('MlpPolicy', env, model_class, n_sampled_goal=4, goal_selection_strategy=goal_selection_strategy, verbose=1) # Train the model model.learn(1000) model.save("./her_bit_env") # WARNING: you must pass an env # or wrap your environment with HERGoalEnvWrapper to use the predict method model = HER.load('./her_bit_env', env=env) obs = env.reset() for _ in range(100): action, _ = model.predict(obs) obs, reward, done, _ = env.step(action)
def callback(_locals, _globals): global n_steps n_steps += 1 if n_steps % 50000 == 0 or n_steps == 10000: print('Saving: ', n_steps) save_path = 'checkpoints/yumi/her/her_{}_task_{}_{}.npy'.format( name, args.task, n_steps) os.makedirs(os.path.dirname(save_path), exist_ok=True) model.save(save_path) return True model = HER('MlpPolicy', env, model_class=DDPG, verbose=1, tensorboard_log=log_dir, **dict(random_exploration=.2)) model.learn(total_timesteps=total_timesteps, callback=callback) model.save("her-yumi-{}-final".format(n_steps)) env.save_running_average(log_dir) obs = env.reset() for i in range(100): action, _states = model.predict(obs) obs, rewards, dones, info = env.step(action) env.render()
# !pip install highway-env # !pip install stable-baselines==2.10.0 # Environment import gym import highway_env # Agent from stable_baselines import HER, SAC """## Training""" env = gym.make("parking-v0") model = HER('MlpPolicy', env, SAC, n_sampled_goal=4, goal_selection_strategy='future', verbose=1, buffer_size=int(1e6), learning_rate=1e-3, gamma=0.9, batch_size=256, policy_kwargs=dict(layers=[256, 256, 256])) model.learn(int(5e4)) """## Visualize a few episodes We first define a simple helper function for visualization of episodes: """ # !pip install gym pyvirtualdisplay # !apt-get install -y xvfb python-opengl ffmpeg from IPython import display as ipythondisplay from pyvirtualdisplay import Display from gym.wrappers import Monitor
from stable_baselines.common.policies import MlpPolicy from stable_baselines import HER, SAC print('setting up environment') #env = gym.make("REALRobot2020-R2J3-v0") env = REALRobotEnv(objects=1) # Currently this wrapper doesn't really return goals but just sample_placeholder # to match the her format. env = GoalWrapper(env, crop_obs=True) print('setting up model') model = HER('MlpPolicy', env, SAC, n_sampled_goal=4, goal_selection_strategy='future', verbose=1, buffer_size=int(1e6), learning_rate=1e-3, gamma=0.95, batch_size=256) print('start learning') model.learn(total_timesteps=256) print('learning done') #Here we need to restart the environent to make rendering possible #(doesn't work with the wrappers right now) env = REALRobotEnv(objects=1) env = GoalWrapper(env, crop_obs=True) env.render("human") print('display model')
def get_goal(self): """Get a random goal depending on the difficulty.""" self.difficulty = np.random.choice(self.difficulties) return move_cube.sample_goal(difficulty=self.difficulty) if __name__ == "__main__": difficulties = np.arange(1, 5) env = gym.make( 'Example_CubeEnv-v0', initializer=Initializer(difficulty=difficulties), action_type=cube_env.ActionType.POSITION, visualization=False, ) model_kwargs = { 'ent_coef': 'auto', 'buffer_size': int(1e6), 'gamma': 0.95, 'learning_starts': 1000, 'train_freq': 1 } model = HER('MlpPolicy', env, SAC, verbose=True, **model_kwargs) model.learn(int(8e6)) model.save("./hersac_CubeEnv_diffall") os.system("shutdown now")
'render': render, 'param_noise': None, 'action_noise': action_noise, 'normalize_observations': normalize, 'nb_train_steps': nb_train_steps, 'nb_rollout_steps': nb_rollout_steps, 'batch_size': batch_size, 'critic_l2_reg': critic_l2_reg, 'buffer_size': buffer_size, 'random_exploration': random_exploration, 'policy_kwargs': { 'layer_norm': True }, 'logging': suff } model = HER('MlpPolicy', env, DDPG, **kwargs) start = time.time() model.learn(total_timesteps=total_timesteps, log_interval=1) if log: model.save("pkl/{}".format(suff)) print( "Saved as {0}, trained {1} primitive policy for {2} timesteps in {3}". format(suff, policy, total_timesteps, time.strftime('%H:%M:%S', time.gmtime(time.time() - start)))) else: print("Trained {0} primitive policy for {1} timesteps in {2}".format( policy, total_timesteps, time.strftime('%H:%M:%S', time.gmtime(time.time() - start))))
def launchAgent(env_name: int, model_name: str, test_mode=False, filepath=None): """ :param test_mode: 에이전트를 테스트 모드로 불러와 주행시킬지를 확인하는 모드입니다. 이럴 시에 학습은 이루어지지 않으며, 주행만 이루어집니다. :param env_name: 불러올 환경의 이름입니다. 1 : 미니맵 이미지를 사용하지 않은, 점 사이의 거리 계산을 한 환경입니다. 2 : 미니맵 이미지를 사용하고, 보상을 업데이트한 모델입니다. 다른 값(기본) : 현재 쓰는 모델입니다. 미니맵 이미지를 사용하고, 보상을 다시 업데이트한 모델입니다. :param model_name: 설정할 모델의 이름입니다. DQN : DQN 모델을 불러옵니다. HER : HER 모델을 불러옵니다. 다른 값(기본) : PPO2 모델을 불러옵니다. :return: 마지막으로 episode를 수행한 모델을 return합니다. """ from stable_baselines import DQN, HER, PPO2 if env_name == 1: from Reinforcement_AI.env.a_env import KartEnv kart_env = KartEnv() policy = "MlpPolicy" elif env_name == 2: from Reinforcement_AI.env.d_image_env import DetailedMiniMapEnv as DetailedMiniMapEnv1 kart_env = DetailedMiniMapEnv1() policy = "CnnPolicy" elif env_name == 3: from Reinforcement_AI.env.a_env2 import KartEnv kart_env = KartEnv() policy = "MlpPolicy" elif env_name == 4: from Reinforcement_AI.env.a_env3 import KartEnv kart_env = KartEnv() policy = "MlpPolicy" else: #env_name == "detailed_minimap_enhanced" or env_name == "4": from Reinforcement_AI.env.e_enhanced_image_env import DetailedMiniMapEnv as DetailedMiniMapEnv2 kart_env = DetailedMiniMapEnv2() policy = "CnnPolicy" if model_name == "DQN": model = DQN(policy=policy, env=kart_env, double_q=True, prioritized_replay=True, verbose=1) elif model_name == "HER": model = HER(policy=policy, env=kart_env, model_class=DQN, verbose=1) else: # model_name == "PPO2" model = PPO2(policy=policy, learning_rate=0.0001, env=kart_env, verbose=1) if test_mode: # 테스트 모드일때 에이전트 불러와서 작동하게함 model.load(filepath) kart_env.set_continuos(True) while True: observation = kart_env.reset() while True: action, _states = model.predict(observation) observation, rewards, dones, info = kart_env.step(action) if dones: break else: for i in range(1000): model.learn(total_timesteps=12500) model.save(str(env_name) + "_" + model_name + "_" + str(i + 1))
import gym import time from stable_baselines.common.policies import MlpPolicy from stable_baselines.common.vec_env import DummyVecEnv from stable_baselines import PPO2, DQN, HER, DDPG import synergyenvs env = gym.make("GraspBoxPybullet-v0") env.render() o = env.reset() # model = PPO2(MlpPolicy, env, verbose=1) model = HER('MlpPolicy', env, DDPG, n_sampled_goal=4, verbose=0) model.load("./her_graspbox-1") env.camera_adjust() for _ in range(10000): env.render() action, _states = model.predict(o) # action = env.action_space.sample() o, r, done, info = env.step(action) print(o, r, done, info) if done: o = env.reset() time.sleep(0.1) env.close()
def train_decision(config=None, save=False, load=False, calender=None, history=None, predict_results_dict=None, test_mode=False, start_date=None, stop_date=None, episode_steps=1000, model='DDPG'): """ 训练决策模型,从数据库读取数据并进行决策训练 参数: config:配置文件, save:保存结果, calender:交易日日历, history:行情信息, all_quotes:拼接之后的行情信息 predict_results_dict:预测结果信息 """ # 首先处理预测数据中字符串日期 MODEL = model predict_dict = {} for k, v in predict_results_dict.items(): assert isinstance(v['predict_date'].iloc[0], str) tmp = v['predict_date'].apply( lambda x: arrow.get(x, 'YYYY-MM-DD').date()) predict_dict[k] = v.rename(index=tmp) env = Portfolio_Prediction_Env(config=config, calender=calender, stock_history=history, window_len=1, prediction_history=predict_dict, start_trade_date=start_date, stop_trade_date=stop_date, save=save) # 测试模式 if test_mode: obs = env.reset() # check_env(env) for i in range(1000): W = np.random.uniform(0.0, 1.0, size=(6, )) offer = np.random.uniform(-10.0, 10.0, size=(6, )) obs, reward, done, infos = env.step(np.hstack((W, offer))) # env.render() if done: env.save_history() break env.close() # 训练模式 if MODEL == "DDPG": # 添加噪声 n_actions = env.action_space.shape param_noise = None # 适合于惯性系统控制的OU噪声 action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions)) model_path = search_file( os.path.join(sys.path[0], 'saved_models', MODEL), MODEL) if len(model_path) > 0 and load: model = DDPG.load( model_path[0], env=env, policy=CustomDDPGPolicy, param_noise=param_noise, action_noise=action_noise, # tensorboard_log='./tb_log', ) else: model = DDPG( policy=CustomDDPGPolicy, env=env, verbose=1, param_noise=param_noise, action_noise=action_noise, # tensorboard_log='./tb_log', ) # 训练步数 model.learn(total_timesteps=episode_steps, ) model.save( os.path.join(sys.path[0], 'saved_models', MODEL, MODEL + '.h5')) elif MODEL == 'TD3': n_actions = env.action_space.shape[-1] # 适合于惯性系统控制的OU噪声 action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions)) model_path = search_file( os.path.join(sys.path[0], 'saved_models', MODEL), MODEL) if len(model_path) > 0 and load: model = TD3.load( model_path[0], env=env, policy=CustomTD3Policy, action_noise=action_noise, # tensorboard_log='./tb_log', ) else: model = TD3( policy=CustomTD3Policy, env=env, verbose=1, action_noise=action_noise, # tensorboard_log='./tb_log', ) # 训练步数 model.learn(total_timesteps=episode_steps, ) model.save( os.path.join(sys.path[0], 'saved_models', MODEL, MODEL + '.h5')) elif MODEL == "HER": """ env必须是GoalEnv """ model_class = DDPG # Available strategies (cf paper): future, final, episode, random goal_selection_strategy = 'future' # equivalent to GoalSelectionStrategy.FUTURE # Wrap the model model = HER(policy=CustomDDPGPolicy, env=env, model_class=model_class, n_sampled_goal=4, goal_selection_strategy=goal_selection_strategy, verbose=1) model.learn(total_timesteps=episode_steps, ) model.save( os.path.join(sys.path[0], 'saved_models', MODEL, MODEL + '.h5')) obs = env.reset() # 实测模式 for i in range(1000): action, _states = model.predict(obs) obs, reward, done, info = env.step(action) # env.render(info=info) if done: if save: env.save_history() env.reset() break env.close()