def run_agent(envs, parameters):
'''Train an agent.'''
alg = parameters['alg']
learning_rate = parameters['learning_rate']
gamma = parameters['gamma']
model_path = parameters['model_path']
set_global_seeds(parameters.get('seed'))
dummy_env = OptVecEnv(envs)
if alg == 'PPO':
model = PPO2(MlpPolicy,
dummy_env,
gamma=gamma,
learning_rate=learning_rate,
verbose=1,
nminibatches=dummy_env.num_envs)
elif alg == 'A2C':
model = A2C(MlpPolicy,
dummy_env,
gamma=gamma,
learning_rate=learning_rate,
verbose=1)
else:
model = DDPG(ddpg.MlpPolicy,
dummy_env,
gamma=gamma,
verbose=1,
actor_lr=learning_rate / 10,
critic_lr=learning_rate)
try:
model.learn(total_timesteps=parameters.get('total_timesteps', 10**6))
except tf.errors.InvalidArgumentError:
LOGGER.error('Possible Nan, %s', str((alg, learning_rate, gamma)))
finally:
dummy_env.close()
model.save(str(model_path))
def run_test(config):
"""Stable baselines test
Mandatory configuration settings:
- 'continuous' agent
- camera_settings enabled
- stable_baselines enabled
"""
env = None
try:
# Create Environment
env = make_env(config)
env = DummyVecEnv([lambda: env])
# Initialize DDPG and start learning
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(
mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions))
model = DDPG(CnnPolicy, env, verbose=1, param_noise=param_noise,
action_noise=action_noise, random_exploration=0.8)
model.learn(total_timesteps=10000)
finally:
if env:
env.close()
else:
clear_carla(config.host, config.port)
print("-----Carla Environment is closed-----")
def test_identity_ddpg():
"""
Test if the algorithm (with a given policy)
can learn an identity transformation (i.e. return observation as an action)
"""
env = DummyVecEnv([lambda: IdentityEnvBox(eps=0.5)])
std = 0.2
param_noise = AdaptiveParamNoiseSpec(initial_stddev=float(std),
desired_action_stddev=float(std))
model = DDPG("MlpPolicy",
env,
gamma=0.0,
param_noise=param_noise,
memory_limit=int(1e6))
model.learn(total_timesteps=20000, seed=0)
n_trials = 1000
reward_sum = 0
set_global_seeds(0)
obs = env.reset()
for _ in range(n_trials):
action, _ = model.predict(obs)
obs, reward, _, _ = env.step(action)
reward_sum += reward
assert reward_sum > 0.9 * n_trials
# Free memory
del model, env
def explore(app,
emulator,
appium,
timesteps,
timer,
save_policy,
policy_dir,
cycle,
nb_train_steps=10,
random_exploration=0.7):
try:
env = TimeFeatureWrapper(app)
model = DDPG(MlpPolicy,
env,
verbose=1,
random_exploration=random_exploration,
nb_train_steps=nb_train_steps)
callback = TimerCallback(timer=timer)
model.learn(total_timesteps=timesteps, callback=callback)
if save_policy:
model.save(f'{policy_dir}{os.sep}{cycle}')
return True
except Exception:
appium.restart_appium()
if emulator is not None:
emulator.restart_emulator()
return False
def test_ddpg_normalization():
"""
Test that observations and returns normalizations are properly saved and loaded.
"""
param_noise = AdaptiveParamNoiseSpec(initial_stddev=0.05,
desired_action_stddev=0.05)
model = DDPG('MlpPolicy',
'Pendulum-v0',
memory_limit=50000,
normalize_observations=True,
normalize_returns=True,
nb_rollout_steps=128,
nb_train_steps=1,
batch_size=64,
param_noise=param_noise)
model.learn(1000)
obs_rms_params = model.sess.run(model.obs_rms_params)
ret_rms_params = model.sess.run(model.ret_rms_params)
model.save('./test_ddpg.zip')
loaded_model = DDPG.load('./test_ddpg.zip')
obs_rms_params_2 = loaded_model.sess.run(loaded_model.obs_rms_params)
ret_rms_params_2 = loaded_model.sess.run(loaded_model.ret_rms_params)
for param, param_loaded in zip(obs_rms_params + ret_rms_params,
obs_rms_params_2 + ret_rms_params_2):
assert np.allclose(param, param_loaded)
del model, loaded_model
if os.path.exists("./test_ddpg.zip"):
os.remove("./test_ddpg.zip")
def train_policy_ddpg(env,
policy,
policy_args,
total_timesteps,
verbose=0,
actor_lr=.5,
critic_lr=.001):
"""
Parameters
----------
env : vectorized set of EncoderWrapper of a TimeLimit wrapper of a restartable env.
policy : ddpg policy class
policy_args : dict of keyword arguments for policy class
total_timesteps : int, how many timesteps to train policy (i.e. 200000)
"""
# the noise objects for DDPG
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=float(0.5) *
np.ones(n_actions))
model = DDPG(policy,
env,
verbose=verbose,
param_noise=param_noise,
action_noise=action_noise,
policy_kwargs=policy_args,
actor_lr=actor_lr,
critic_lr=critic_lr)
#model = PPO2(policy, env)
model.learn(total_timesteps)
return model
def main(env: PSMCartesianDDPGEnv):
# the noise objects for DDPG
n_actions = env.action.action_space.shape[0]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=float(0.5) *
np.ones(n_actions))
model = DDPG(MlpPolicy,
env,
gamma=0.95,
verbose=1,
nb_train_steps=300,
nb_rollout_steps=150,
param_noise=param_noise,
batch_size=128,
action_noise=action_noise,
random_exploration=0.05,
normalize_observations=True,
tensorboard_log="./ddpg_dvrk_tensorboard/",
observation_range=(-1.5, 1.5),
critic_l2_reg=0.01)
model.learn(total_timesteps=4000000,
log_interval=100,
callback=CheckpointCallback(
save_freq=100000, save_path="./ddpg_dvrk_tensorboard/"))
model.save("./ddpg_robot_env")
def main():
# unpause Simulation so that robot receives data on all topics
gazebo_connection.GazeboConnection().unpauseSim()
# create node
rospy.init_node('pickbot_gym', anonymous=True, log_level=rospy.FATAL)
env = gym.make('Pickbot-v1')
# the noise objects for DDPG
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=float(0.5) *
np.ones(n_actions))
model = DDPG(MlpPolicy,
env,
verbose=1,
param_noise=param_noise,
action_noise=action_noise)
model.learn(total_timesteps=200000)
print("Saving model to pickbot_model_ddpg_continuous_" + timestamp +
".pkl")
model.save("pickbot_model_ddpg_continuous_" + timestamp)
def ddpg(env_id,
timesteps,
policy="MlpPolicy",
log_interval=None,
tensorboard_log=None,
seed=None,
load_weights=None):
from stable_baselines import DDPG
env = gym.make(env_id)
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=float(0.5) *
np.ones(n_actions))
if load_weights is not None:
model = DDPG.load(load_weights, env=env)
else:
model = DDPG(policy,
env,
verbose=1,
param_noise=param_noise,
action_noise=action_noise,
tensorboard_log=tensorboard_log)
callback = WandbRenderEnvCallback(model_name="ddpg", env_name=env_id)
model.learn(total_timesteps=timesteps,
log_interval=log_interval,
callback=callback)
save_model_weights(model, "ddpg", env_id, policy, seed=seed, path=".")
def main():
# create Environment
env = iCubPushGymEnv(urdfRoot=robot_data.getDataPath(), renders=False, useIK=1,
isDiscrete=0, rnd_obj_pose=0, maxSteps=2000, reward_type=0)
# set seed
seed = 1
tf.reset_default_graph()
set_global_seed(seed)
env.seed(seed)
# set log
monitor_dir = os.path.join(log_dir,'log')
os.makedirs(monitor_dir, exist_ok=True)
env = Monitor(env, monitor_dir+'/', allow_early_resets=True)
# create agent model
nb_actions = env.action_space.shape[-1]
action_noise = NormalActionNoise(mean=np.zeros(nb_actions), sigma=float(0.5373) * np.ones(nb_actions))
model = DDPG('LnMlpPolicy', env, action_noise=action_noise, gamma=0.99, batch_size=16,
normalize_observations=True,normalize_returns=False, memory_limit=100000,
verbose=1, tensorboard_log=os.path.join(log_dir,'tb'),full_tensorboard_log=False)
#start learning
model.learn(total_timesteps=500000, seed=seed, callback=callback)
# save model
print("Saving model.pkl to ",log_dir)
act.save(log_dir+"/final_model.pkl")
def train_DDPG(self, model_name, model_params=config.DDPG_PARAMS):
"""DDPG model"""
from stable_baselines import DDPG
from stable_baselines.ddpg.policies import DDPGPolicy
from stable_baselines.common.noise import OrnsteinUhlenbeckActionNoise
env_train = self.env
n_actions = env_train.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=float(0.5) *
np.ones(n_actions))
start = time.time()
model = DDPG('MlpPolicy',
env_train,
batch_size=model_params['batch_size'],
buffer_size=model_params['buffer_size'],
param_noise=param_noise,
action_noise=action_noise,
verbose=model_params['verbose'])
model.learn(total_timesteps=model_params['timesteps'])
end = time.time()
model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}")
print('Training time (DDPG): ', (end - start) / 60, ' minutes')
return model
def train_DDPG(env_train, model_name, timesteps=50000):
"""DDPG model"""
start = time.time()
model = DDPG('MlpPolicy', env_train)
model.learn(total_timesteps=timesteps)
end = time.time()
model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}")
print('Training time (DDPG): ', (end - start) / 60, ' minutes')
return model
def DDPGAgent(multi_stock_env, num_episodes):
models_folder = 'saved_models'
rewards_folder = 'saved_rewards'
env = DummyVecEnv([lambda: multi_stock_env])
# the noise objects for DDPG
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions))
# Hyper parameters
GAMMA = 0.99
TAU = 0.001
BATCH_SIZE = 16
ACTOR_LEARNING_RATE = 0.0001
CRITIC_LEARNING_RATE = 0.001
BUFFER_SIZE = 500
print("\nRunning DDPG Agent...\n")
model = DDPG(MlpPolicy, env,
gamma = GAMMA, tau = TAU, batch_size = BATCH_SIZE,
actor_lr = ACTOR_LEARNING_RATE, critic_lr = CRITIC_LEARNING_RATE,
buffer_size = BUFFER_SIZE, verbose=1,
param_noise=param_noise, action_noise=action_noise)
model.learn(total_timesteps=50000)
model.save(f'{models_folder}/rl/ddpg.h5')
del model
model = DDPG.load(f'{models_folder}/rl/ddpg.h5')
obs = env.reset()
portfolio_value = []
for e in range(num_episodes):
action, _states = model.predict(obs)
next_state, reward, done, info = env.step(action)
print(f"episode: {e + 1}/{num_episodes}, episode end value: {info[0]['cur_val']:.2f}")
portfolio_value.append(round(info[0]['cur_val'], 3))
# save portfolio value for each episode
np.save(f'{rewards_folder}/rl/ddpg.npy', portfolio_value)
print("\nDDPG Agent run complete and saved!")
a = np.load(f'./saved_rewards/rl/ddpg.npy')
print(f"\nCumulative Portfolio Value Average reward: {a.mean():.2f}, Min: {a.min():.2f}, Max: {a.max():.2f}")
plt.plot(a)
plt.title("Portfolio Value Per Episode (DDPG)")
plt.ylabel("Portfolio Value")
plt.xlabel("Episodes")
plt.show()
def train_ddpg():
env = gimbal(5, 500)
env = DummyVecEnv([lambda: env])
eval_env = gimbal(5, 500)
eval_env = DummyVecEnv([lambda: eval_env])
# the noise objects for DDPG
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = None
model = DDPG(policy=MlpPolicy,
env=env,
gamma=0.99,
memory_policy=None,
eval_env=eval_env,
nb_train_steps=500,
nb_rollout_steps=500,
nb_eval_steps=500,
param_noise=param_noise,
action_noise=action_noise,
normalize_observations=False,
tau=0.001,
batch_size=128,
param_noise_adaption_interval=50,
normalize_returns=False,
enable_popart=False,
observation_range=(-5000.0, 5000.0),
critic_l2_reg=0.0,
return_range=(-inf, inf),
actor_lr=0.0001,
critic_lr=0.001,
clip_norm=None,
reward_scale=1.0,
render=False,
render_eval=False,
memory_limit=50000,
verbose=1,
tensorboard_log="./logs",
_init_setup_model=True,
policy_kwargs=None,
full_tensorboard_log=False)
#model = DDPG.load("./models/baseline_ddpg_t2")
#model.set_env(env)
model.learn(total_timesteps=1000000,
callback=None,
seed=None,
log_interval=100,
tb_log_name='DDPG',
reset_num_timesteps=True)
model.save("./models/baseline_ddpg_t2")
def test_ddpg_eval_env():
"""
Additional test to check that everything is working when passing
an eval env.
"""
eval_env = gym.make("Pendulum-v0")
model = DDPG("MlpPolicy",
"Pendulum-v0",
nb_rollout_steps=5,
nb_train_steps=2,
nb_eval_steps=10,
eval_env=eval_env,
verbose=0)
model.learn(1000)
def run(self):
self._init()
env = self.env
model = self.model
objective = self.objective
if objective == "infogain":
wenv = InfogainEnv(env, model)
elif objective == "prederr":
wenv = PrederrEnv(env, model)
else:
raise AttributeError(
"Objective '{}' is unknown. Needs to be 'infogain' or 'prederr'"
.format(objective))
wenv.max_episode_len = self.horizon
wenv.end_episode_callback = self._end_episode
dvenv = DummyVecEnv([lambda: wenv])
if self.rl_algo == "ddpg":
self.logger.info("Setting up DDPG as model-free RL algorithm.")
pn = AdaptiveParamNoiseSpec()
an = NormalActionNoise(np.array([0]), np.array([1]))
rl_model = DDPG(DDPGMlpPolicy,
dvenv,
verbose=1,
render=False,
action_noise=an,
param_noise=pn,
nb_rollout_steps=self.horizon,
nb_train_steps=self.horizon)
elif self.rl_algo == "sac":
self.logger.info("Setting up SAC as model-free RL algorithm.")
rl_model = SAC(SACMlpPolicy,
dvenv,
verbose=1,
learning_starts=self.horizon)
else:
raise AttributeError(
"Model-free RL algorithm '{}' is unknown.".format(
self.rl_algo))
# Train the agent
max_steps_total = self.horizon * self.n_episodes * 100
try:
self.logger.info("Start the agent")
rl_model.learn(total_timesteps=max_steps_total, seed=self.seed)
except MaxEpisodesReachedException:
print("Exploration finished.")
def train_DDPG(env_train, model_name, timesteps=10000):
"""DDPG model"""
# the noise objects for DDPG
n_actions = env_train.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions))
start = time.time()
model = DDPG('MlpPolicy', env_train, param_noise=param_noise, action_noise=action_noise)
model.learn(total_timesteps=timesteps)
end = time.time()
model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}")
print('Training time (DDPG): ', (end - start) / 60, ' minutes')
return model
def __call__(self, trial):
# Calculate an objective value by using the extra arguments.
env_id = 'gym_custom:fooCont-v0'
env = gym.make(env_id, data=self.train_data)
env = DummyVecEnv([lambda: env])
algo = trial.suggest_categorical('algo', ['TD3'])
model = 0
if algo == 'PPO2':
policy_choice = trial.suggest_categorical('policy', [False, True])
policy = commonMlp if policy_choice else commonMlpLstm
model_params = optimize_ppo2(trial)
model = PPO2(policy, env, verbose=0, nminibatches=1, **model_params)
model.learn(276*7000)
elif algo == 'DDPG':
policy_choice = trial.suggest_categorical('policy', [False, True])
policy = ddpgLnMlp
model_params = sample_ddpg_params(trial)
model= DDPG(policy, env, verbose=0, **model_params)
model.learn(276*7000)
elif algo == 'TD3':
policy_choice = trial.suggest_categorical('policy', [False, True])
policy = td3MLP if policy_choice else td3LnMlp
model_params = sample_td3_params(trial)
model = TD3(policy, env, verbose=0, **model_params)
model.learn(276*7000*3)
rewards = []
reward_sum = 0.0
env = gym.make(env_id, data=self.test_data)
env = DummyVecEnv([lambda: env])
obs = env.reset()
for ep in range(1000):
for step in range(276):
action, _ = model.predict(obs)
obs, reward, done, _ = env.step(action)
reward_sum += reward
if done:
rewards.append(reward_sum)
reward_sum = 0.0
obs = env.reset()
def main(output_folder_path:Path):
# Set gym-carla environment
agent_config = AgentConfig.parse_file(Path("configurations/agent_configuration.json"))
carla_config = CarlaConfig.parse_file(Path("configurations/carla_configuration.json"))
params = {
"agent_config": agent_config,
"carla_config": carla_config,
"ego_agent_class": RLPIDAgent,
"max_collision": 5
}
env = gym.make('roar-pid-v0', params=params)
env.reset()
model_params: dict = {
"verbose": 1,
"render": True,
"tensorboard_log": (output_folder_path / "tensorboard").as_posix()
}
latest_model_path = find_latest_model(output_folder_path)
if latest_model_path is None:
model = DDPG(LnMlpPolicy, env=env, **model_params) # full tensorboard log can take up space quickly
else:
model = DDPG.load(latest_model_path, env=env, **model_params)
model.render = True
model.tensorboard_log = (output_folder_path / "tensorboard").as_posix()
logging_callback = LoggingCallback(model=model)
checkpoint_callback = CheckpointCallback(save_freq=1000, verbose=2, save_path=(output_folder_path / "checkpoints").as_posix())
event_callback = EveryNTimesteps(n_steps=100, callback=checkpoint_callback)
callbacks = CallbackList([checkpoint_callback, event_callback, logging_callback])
model = model.learn(total_timesteps=int(1e10), callback=callbacks, reset_num_timesteps=False)
model.save(f"pid_ddpg_{datetime.now()}")
def main():
env1 = KukaDiverseObjectEnv(renders=True, isDiscrete=False)
model = DDPG(MlpPolicy, env1, verbose=1)
# = deepq.models.mlp([64])
model.learn(total_timesteps=500000)
#max_timesteps=10000000,
# exploration_fraction=0.1,
# exploration_final_eps=0.02,
# print_freq=10,
# callback=callback, network='mlp')
print("Saving model to kukadiverse_model.pkl")
model.save("kukadiversecont_model.pkl")
main()
def test_ddpg_popart():
"""
Test DDPG with pop-art normalization
"""
n_actions = 1
action_noise = NormalActionNoise(mean=np.zeros(n_actions),
sigma=0.1 * np.ones(n_actions))
model = DDPG('MlpPolicy',
'Pendulum-v0',
memory_limit=50000,
normalize_observations=True,
normalize_returns=True,
nb_rollout_steps=128,
nb_train_steps=1,
batch_size=64,
action_noise=action_noise,
enable_popart=True)
model.learn(1000)
def ppo1_nmileg_pool(sensory_value):
RL_method = "PPO1"
# total_MC_runs = 50
experiment_ID = "handtest_rot_pool_with_MC_C_task0/"
save_name_extension = RL_method
total_timesteps = 500000
sensory_info = "sensory_{}".format(sensory_value)
current_mc_run_num =22 #starts from 0
for mc_cntr in range(current_mc_run_num, current_mc_run_num+1):
log_dir = "./logs/{}/MC_{}/{}/{}/".format(experiment_ID, mc_cntr, RL_method, sensory_info)
# defining the environments
env = gym.make('HandManipulate-v1{}'.format(sensory_value))
#env = gym.wrappers.Monitor(env, "./tmp/gym-results", video_callable=False, force=True)
## setting the Monitor
env = gym.wrappers.Monitor(env, log_dir+"Monitor/", video_callable=False, force=True, uid="Monitor_info")
# defining the initial model
if RL_method == "PPO1":
model = PPO1(common_MlpPolicy, env, verbose=1, tensorboard_log=log_dir)
elif RL_method == "PPO2":
env = DummyVecEnv([lambda: env])
model = PPO2(common_MlpPolicy, env, verbose=1, tensorboard_log=log_dir)
elif RL_method == "DDPG":
env = DummyVecEnv([lambda: env])
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5)* 5 * np.ones(n_actions))
model = DDPG(DDPG_MlpPolicy, env, verbose=1, param_noise=param_noise, action_noise=action_noise, tensorboard_log=log_dir)
else:
raise ValueError("Invalid RL mode")
# setting the environment on the model
#model.set_env(env)
# setting the random seed for some of the random instances
random_seed = mc_cntr
random.seed(random_seed)
env.seed(random_seed)
env.action_space.seed(random_seed)
np.random.seed(random_seed)
tf.random.set_random_seed(random_seed)
# training the model
# training the model
model.learn(total_timesteps=total_timesteps)
# saving the trained model
model.save(log_dir+"/model")
return None
def train_identity_ddpg():
env = DummyVecEnv([lambda: IdentityEnvBox(eps = 0.5)])
std = 0.2
param_noise = AdaptiveParamNoiseSpec(initial_stddev=float(std), desired_action_stddev=float(std))
model = DDPG("MlpPolicy", env, gamma=0.0, param_noise=param_noise, memory_limit=int(1e6))
model.learn(total_timesteps=20000, seed=0)
n_trials = 1000
reward_sum = 0
set_global_seeds(0)
obs = env.reset()
for _ in range(n_trials):
action, _ = model.predict(obs)
obs, reward, _, _ = env.step(action)
reward_sum += reward
assert reward_sum > 0.9 * n_trials
del model, env
def train_DDPG(self, model_name, ddpg_params=config.DDPG_PARAMS):
"""DDPG model"""
from stable_baselines import DDPG
from stable_baselines.ddpg.policies import DDPGPolicy
from stable_baselines.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise, AdaptiveParamNoiseSpec
env_train = self.env
start = time.time()
model = DDPG('MlpPolicy',
env_train,
batch_size=ddpg_params['batch_size'],
buffer_size=ddpg_params['buffer_size'],
verbose=ddpg_params['verbose'])
model.learn(total_timesteps=ddpg_params['timesteps'])
end = time.time()
model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}")
print('Training time (DDPG): ', (end - start) / 60, ' minutes')
return model
def train_policy(num_of_envs, log_relative_path, maximum_episode_length,
skip_frame, seed_num, ddpg_config, total_time_steps,
validate_every_timesteps, task_name):
print("Using MPI for multiprocessing with {} workers".format(
MPI.COMM_WORLD.Get_size()))
rank = MPI.COMM_WORLD.Get_rank()
print("Worker rank: {}".format(rank))
task = generate_task(task_generator_id=task_name,
dense_reward_weights=np.array(
[250, 0, 125, 0, 750, 0, 0, 0.005]),
fractional_reward_weight=1,
goal_height=0.15,
tool_block_mass=0.02)
env = CausalWorld(task=task,
skip_frame=skip_frame,
enable_visualization=False,
seed=0,
max_episode_length=maximum_episode_length,
normalize_actions=False,
normalize_observations=False)
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=float(0.5) *
np.ones(n_actions))
policy_kwargs = dict(layers=[256, 256])
checkpoint_callback = CheckpointCallback(save_freq=int(
validate_every_timesteps / num_of_envs),
save_path=log_relative_path,
name_prefix='model')
model = DDPG(MlpPolicy,
env,
verbose=2,
param_noise=param_noise,
action_noise=action_noise,
policy_kwargs=policy_kwargs,
**ddpg_config)
model.learn(total_timesteps=total_time_steps,
tb_log_name="ddpg",
callback=checkpoint_callback)
return
def run_baseline_ddpg(env_name, train=True):
import numpy as np
# from stable_baselines.ddpg.policies import MlpPolicy
from stable_baselines.common.vec_env import DummyVecEnv
from stable_baselines.ddpg.noise import OrnsteinUhlenbeckActionNoise
from stable_baselines import DDPG
env = gym.make(env_name)
env = DummyVecEnv([lambda: env])
if train:
# mlp
from stable_baselines.ddpg.policies import FeedForwardPolicy
class CustomPolicy(FeedForwardPolicy):
def __init__(self, *args, **kwargs):
super(CustomPolicy, self).__init__(*args, **kwargs,
layers=[64, 64, 64],
layer_norm=True,
feature_extraction="mlp")
# the noise objects for DDPG
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions)+0.15, sigma=0.3 * np.ones(n_actions))
model = DDPG(CustomPolicy, env, verbose=1, param_noise=param_noise, action_noise=action_noise,
tau=0.01, observation_range=(env.observation_space.low, env.observation_space.high),
critic_l2_reg=0, actor_lr=1e-3, critic_lr=1e-3, memory_limit=100000)
model.learn(total_timesteps=1e5)
model.save("checkpoints/ddpg_" + env_name)
else:
model = DDPG.load("checkpoints/ddpg_" + env_name)
obs = env.reset()
while True:
action, _states = model.predict(obs)
obs, rewards, dones, info = env.step(action)
env.render()
print("state: ", obs, " reward: ", rewards, " done: ", dones, "info: ", info)
del model # remove to demonstrate saving and loading
def main(args):
#Starting the timer to record the operation time.
start = time.time()
env_id = 'fwmav_hover-v0'
#Creating a vector of size 1 which only has the environment.
env = DummyVecEnv([make_env(env_id, 0)])
# env = SubprocVecEnv([make_env(env_id, i) for i in range(args.n_cpu)])
# -1 argument means the shape will be found automatically.
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=float(0.5) *
np.ones(n_actions))
model = DDPG(
policy=MyDDPGPolicy,
env=env,
gamma=1.0,
nb_train_steps=5000,
nb_rollout_steps=10000,
nb_eval_steps=10000,
param_noise=param_noise,
action_noise=action_noise,
tau=0.003,
batch_size=256,
observation_range=(-np.inf, np.inf),
actor_lr=0.0001,
critic_lr=0.001,
reward_scale=0.05,
memory_limit=10000000,
verbose=1,
)
model.learn(total_timesteps=args.time_step)
model.save(args.model_path)
#End timer.
end = time.time()
print("Time used: ", end - start)
def training(env):
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=float(0.5) *
np.ones(n_actions))
model = DDPG(MlpPolicy,
env,
verbose=1,
param_noise=param_noise,
action_noise=action_noise,
render=True,
return_range=[-1.0, 1.0],
observation_range=[-2.0, 2.0])
model.learn(total_timesteps=40000)
time = datetime.now().strftime("%m%d_%H%M%S")
model.save("models\\ddpg_sbl_" + time)
del model # remove to demonstrate saving and loading
testing(env, time)
def ppo1_nmileg_pool(stiffness_value):
RL_method = "PPO1"
experiment_ID = "experiment_4_pool_A/mc_1/"
save_name_extension = RL_method
total_timesteps = 500000
stiffness_value_str = "stiffness_{}".format(stiffness_value)
log_dir = "./logs/{}/{}/{}/".format(experiment_ID, RL_method,
stiffness_value_str)
# defining the environments
env = gym.make('TSNMILeg{}-v1'.format(stiffness_value))
#env = gym.wrappers.Monitor(env, "./tmp/gym-results", video_callable=False, force=True)
# defining the initial model
if RL_method == "PPO1":
model = PPO1(common_MlpPolicy, env, verbose=1, tensorboard_log=log_dir)
elif RL_method == "PPO2":
env = DummyVecEnv([lambda: env])
model = PPO2(common_MlpPolicy, env, verbose=1, tensorboard_log=log_dir)
elif RL_method == "DDPG":
env = DummyVecEnv([lambda: env])
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=float(0.5) * 5 *
np.ones(n_actions))
model = DDPG(DDPG_MlpPolicy,
env,
verbose=1,
param_noise=param_noise,
action_noise=action_noise,
tensorboard_log=log_dir)
else:
raise ValueError("Invalid RL mode")
# setting the environment on the model
#model.set_env(env)
# training the model
# training the model
model.learn(total_timesteps=total_timesteps)
# saving the trained model
model.save(log_dir + "/model")
return None
def optimize_agent(trial):
""" Train the model and optimise
Optuna maximises the negative log likelihood, so we
need to negate the reward here
"""
model_params = optimize_ddpg(trial)
seed = trial.suggest_int('numpyseed', 1, 429496729)
np.random.seed(seed)
original_env = gym.make('rustyblocks-v0')
original_env.max_invalid_tries = 3
env = DummyVecEnv([lambda: original_env])
model = DDPG("MlpPolicy", env, verbose=0, observation_range=(-126,126), **model_params)
print("DOING LEARING a2c")
original_env.force_progression = False
model.learn(int(2e4*5), seed=seed)
print("DONE LEARING a2c")
original_env.max_invalid_tries = -1
rewards = []
n_episodes, reward_sum = 0, 0.0
obs = env.reset()
original_env.force_progression = True
original_env.invalid_try_limit = 5000
while n_episodes < 4:
action, _ = model.predict(obs)
obs, reward, done, _ = env.step(action)
reward_sum += reward
if done:
rewards.append(reward_sum)
reward_sum = 0.0
n_episodes += 1
obs = env.reset()
last_reward = np.mean(rewards)
trial.report(last_reward)
return last_reward
