def run(*_):
env = normalize(
MultiGoalEnv(
actuation_cost_coeff=1,
distance_cost_coeff=0.1,
goal_reward=1,
init_sigma=0.1,
))
pool = SimpleReplayBuffer(
max_replay_buffer_size=1e6,
env_spec=env.spec,
)
base_kwargs = dict(min_pool_size=30,
epoch_length=1000,
n_epochs=1000,
max_path_length=30,
batch_size=64,
n_train_repeat=1,
eval_render=True,
eval_n_episodes=10,
eval_deterministic=True)
M = 100
qf = NNQFunction(env_spec=env.spec, hidden_layer_sizes=[M, M])
vf = NNVFunction(env_spec=env.spec, hidden_layer_sizes=[M, M])
policy = GMMPolicy(env_spec=env.spec,
K=4,
hidden_layer_sizes=[M, M],
qf=qf,
reg=0.001)
plotter = QFPolicyPlotter(qf=qf,
policy=policy,
obs_lst=np.array([[-2.5, 0.0], [0.0, 0.0],
[2.5, 2.5]]),
default_action=[np.nan, np.nan],
n_samples=100)
algorithm = SAC(base_kwargs=base_kwargs,
env=env,
policy=policy,
pool=pool,
qf=qf,
vf=vf,
plotter=plotter,
lr=3E-4,
scale_reward=3,
discount=0.99,
tau=0.001,
save_full_state=True)
algorithm.train()
def run_experiment(variant):
env_params = variant['env_params']
policy_params = variant['policy_params']
value_fn_params = variant['value_fn_params']
algorithm_params = variant['algorithm_params']
replay_buffer_params = variant['replay_buffer_params']
sampler_params = variant['sampler_params']
task = variant['task'] #what does task and domain relate to ??
domain = variant['domain']
env = normalize(ENVIRONMENTS[domain][task](**env_params)) #TODO- define the baxter environment
pool = SimpleReplayBuffer(env_spec=env.spec, **replay_buffer_params) # TODO:Need to have a HER Pool
sampler = SimpleSampler(**sampler_params)
base_kwargs = dict(algorithm_params['base_kwargs'], sampler=sampler)
M = value_fn_params['layer_size']
qf1 = NNQFunction(env_spec=env.spec, hidden_layer_sizes=(M, M), name='qf1')
qf2 = NNQFunction(env_spec=env.spec, hidden_layer_sizes=(M, M), name='qf2')
vf = NNVFunction(env_spec=env.spec, hidden_layer_sizes=(M, M))
initial_exploration_policy = UniformPolicy(env_spec=env.spec)
if policy_params['type'] == 'gaussian':
policy = GaussianPolicy(
env_spec=env.spec,
hidden_layer_sizes=(M,M),
reparameterize=policy_params['reparameterize'],
reg=1e-3,
)
elif policy_params['type'] == 'lsp':
nonlinearity = {
None: None,
'relu': tf.nn.relu,
'tanh': tf.nn.tanh
}[policy_params['preprocessing_output_nonlinearity']]
preprocessing_hidden_sizes = policy_params.get('preprocessing_hidden_sizes')
if preprocessing_hidden_sizes is not None:
observations_preprocessor = MLPPreprocessor(
env_spec=env.spec,
layer_sizes=preprocessing_hidden_sizes,
output_nonlinearity=nonlinearity)
else:
observations_preprocessor = None
policy_s_t_layers = policy_params['s_t_layers']
policy_s_t_units = policy_params['s_t_units']
s_t_hidden_sizes = [policy_s_t_units] * policy_s_t_layers
bijector_config = {
'num_coupling_layers': policy_params['coupling_layers'],
'translation_hidden_sizes': s_t_hidden_sizes,
'scale_hidden_sizes': s_t_hidden_sizes,
}
policy = LatentSpacePolicy(
env_spec=env.spec,
squash=policy_params['squash'],
bijector_config=bijector_config,
reparameterize=policy_params['reparameterize'],
q_function=qf1,
observations_preprocessor=observations_preprocessor)
elif policy_params['type'] == 'gmm':
# reparameterize should always be False if using a GMMPolicy
policy = GMMPolicy(
env_spec=env.spec,
K=policy_params['K'],
hidden_layer_sizes=(M, M),
reparameterize=policy_params['reparameterize'],
qf=qf1,
reg=1e-3,
)
else:
raise NotImplementedError(policy_params['type'])
algorithm = SAC(
base_kwargs=base_kwargs,
env=env,
policy=policy,
initial_exploration_policy=initial_exploration_policy,
pool=pool,
qf1=qf1,
qf2=qf2,
vf=vf,
lr=algorithm_params['lr'],
scale_reward=algorithm_params['scale_reward'],
discount=algorithm_params['discount'],
tau=algorithm_params['tau'],
reparameterize=algorithm_params['reparameterize'],
target_update_interval=algorithm_params['target_update_interval'],
action_prior=policy_params['action_prior'],
save_full_state=False,
)
algorithm._sess.run(tf.global_variables_initializer())
algorithm.train()
def main(trial,
use_optuna,
env,
seed,
entropy_coeff,
n_epochs,
dynamic_coeff,
clip_norm,
normalize_obs,
buffer_size,
max_path_length,
min_pool_size,
batch_size,
policy_mode,
eval_model,
eval_n_episodes,
eval_n_frequency,
exploitation_ratio,
return_queue=None,
scale_reward=1.):
if use_optuna:
logger.configure(logger.get_dir(),
log_suffix="_optune{}".format(trial.number),
enable_std_out=False)
logger.set_level(logger.DISABLED)
tf.set_random_seed(seed=seed)
env.min_action = env.action_space.low[0]
env.max_action = env.action_space.high[0]
if hasattr(env, "seed"):
env.seed(seed)
else:
env.env.seed(seed)
# define value function
layer_size = 100
qf = NNQFunction(env_spec=env.spec,
hidden_layer_sizes=(layer_size, layer_size))
vf = NNVFunction(env_spec=env.spec,
hidden_layer_sizes=(layer_size, layer_size))
# use GMM policy
if policy_mode == "GMMPolicy":
# use GMM policy
policy = GMMPolicy(env_spec=env.spec,
K=4,
hidden_layer_sizes=[layer_size, layer_size],
qf=qf,
reg=1e-3,
squash=True)
elif policy_mode == "EExploitation":
policy = EExploitationPolicy(
env_spec=env.spec,
K=4,
hidden_layer_sizes=[layer_size, layer_size],
qf=qf,
reg=1e-3,
squash=True,
e=exploitation_ratio)
else:
if policy_mode == "Knack-exploration" or policy_mode == "kurtosis":
metric = "kurtosis"
elif policy_mode in [
"signed_variance", "negative_signed_variance",
"small_variance", "large_variance"
]:
metric = policy_mode
elif "kurtosis-" in policy_mode:
metric = policy_mode
else:
raise AssertionError(
"policy_mode should be GMMPolicy or Knack-exploration or Knack-exploration or signed_variance or variance"
)
policy = KnackBasedPolicy(
env_spec=env.spec,
K=4,
hidden_layer_sizes=[layer_size, layer_size],
qf=qf,
vf=vf,
reg=1e-3,
squash=True,
metric=metric,
exploitation_ratio=exploitation_ratio,
optuna_trial=trial,
)
# TODO
base_kwargs = dict(
epoch_length=1000,
n_epochs=n_epochs,
# scale_reward=1,
n_train_repeat=1,
eval_render=False,
eval_n_episodes=eval_n_episodes,
eval_deterministic=True,
eval_n_frequency=eval_n_frequency)
max_replay_buffer_size = buffer_size
pool = SimpleReplayBuffer(env_spec=env.spec,
max_replay_buffer_size=max_replay_buffer_size)
sampler_params = {
'max_path_length': max_path_length,
'min_pool_size': min_pool_size,
'batch_size': batch_size
}
sampler = NormalizeSampler(
**sampler_params) if normalize_obs else SimpleSampler(**sampler_params)
base_kwargs = dict(base_kwargs, sampler=sampler)
algorithm = SAC(base_kwargs=base_kwargs,
env=env,
policy=policy,
pool=pool,
qf=qf,
vf=vf,
lr=3e-4,
scale_reward=scale_reward,
discount=0.99,
tau=1e-2,
target_update_interval=1,
action_prior='uniform',
save_full_state=False,
dynamic_coeff=dynamic_coeff,
entropy_coeff=entropy_coeff,
clip_norm=clip_norm)
algorithm._sess.run(tf.global_variables_initializer())
if eval_model is None:
avg_return = algorithm.train()
if return_queue is not None:
return_queue.put(avg_return)
tf.reset_default_graph()
algorithm._sess.close()
del algorithm
return avg_return
else:
return algorithm
def run_experiment(variant):
env_params = variant['env_params']
policy_params = variant['policy_params']
value_fn_params = variant['value_fn_params']
algorithm_params = variant['algorithm_params']
replay_buffer_params = variant['replay_buffer_params']
sampler_params = variant['sampler_params']
task = variant['task']
domain = variant['domain']
constants.COST_TYPE = variant['algorithm_params']['cost_type']
register(
id='MECS-v1',
entry_point='sac.envs.environment_V_sweep:MEC_v1',
max_episode_steps=5000,
)
register(
id='MECS-v2',
entry_point='sac.envs.env_V_sweep_v2:MEC_v2',
max_episode_steps=5000,
)
register(
id='MECS-v3',
entry_point='sac.envs.env_V_sweep_v3:MEC_v3',
max_episode_steps=5000,
)
register(
id='MECS-v4',
entry_point='sac.envs.env_V_sweep_v4:MEC_v4',
max_episode_steps=5000,
)
register(
id='MECS-v5',
entry_point='sac.envs.env_V_sweep_v5:MEC_v5',
max_episode_steps=5000,
)
register(
id='MECS-v6',
entry_point='sac.envs.env_V_sweep_v6:MEC_v6',
max_episode_steps=5000,
)
register(
id='MECS-v61',
entry_point='sac.envs.env_V_sweep_v6_with_a:MEC_v6',
max_episode_steps=5000,
)
register(
id='MECS-v7',
entry_point='sac.envs.env_V_sweep_v7_new:MEC_v7',
max_episode_steps=5000,
)
register(
id='MECS-v8',
entry_point='sac.envs.env_V_sweep_v8_new:MEC_v8',
max_episode_steps=5000,
)
register(
id='MECS-v9',
entry_point='sac.envs.env_V_sweep_v9:MEC_v9',
max_episode_steps=5000,
)
register(
id='MECS-v10',
entry_point='sac.envs.env_V_sweep_v10:MEC_v10',
max_episode_steps=5000,
)
register(
id='MECS-v11',
entry_point='sac.envs.env_V_sweep_v11:MEC_v11',
max_episode_steps=5000,
)
register(
id='MECS-v12',
entry_point='sac.envs.env_V_sweep_v12:MEC_v12',
max_episode_steps=5000,
)
register(
id='MECS-v13',
entry_point='sac.envs.env_V_sweep_v13:MEC_v13',
max_episode_steps=5000,
)
register(
id='MECS-v14',
entry_point='sac.envs.env_V_sweep_v14:MEC_v14',
max_episode_steps=5000,
)
register(
id='MECS-v15',
entry_point='sac.envs.env_V_sweep_v15:MEC_v15',
max_episode_steps=5000,
)
register(
id='MECS-v16',
entry_point='sac.envs.env_V_sweep_v16:MEC_v16',
max_episode_steps=5000,
)
register(
id='MECS-v17',
entry_point='sac.envs.env_V_sweep_v17:MEC_v17',
max_episode_steps=5000,
)
register(
id='MECS-v18',
entry_point='sac.envs.env_V_sweep_v18:MEC_v18',
max_episode_steps=5000,
)
register(
id='MECS-v19',
entry_point='sac.envs.env_V_sweep_v19:MEC_v19',
max_episode_steps=5000,
)
register(
id='MECS-v20',
entry_point='sac.envs.env_V_sweep_v20:MEC_v20',
max_episode_steps=5000,
)
register(
id='MECS-v21',
entry_point='sac.envs.env_V_sweep_v21:MEC_v21',
max_episode_steps=5000,
)
register(
id='MECS-v22',
entry_point='sac.envs.env_V_sweep_v22:MEC_v22',
max_episode_steps=5000,
)
register(
id='MECS-v23',
entry_point='sac.envs.env_V_sweep_v23:MEC_v23',
max_episode_steps=5000,
)
register(
id='MECS-v24',
entry_point='sac.envs.env_V_sweep_v24:MEC_v24',
max_episode_steps=5000,
)
register(
id='MECS-v25',
entry_point='sac.envs.env_V_sweep_v25:MEC_v25',
max_episode_steps=5000,
)
register(
id='MECS-v26',
entry_point='sac.envs.env_V_sweep_v26:MEC_v26',
max_episode_steps=5000,
)
register(
id='MECS-v27',
entry_point='sac.envs.env_V_sweep_v27:MEC_v27',
max_episode_steps=5000,
)
register(
id='MECS-v28',
entry_point='sac.envs.env_V_sweep_v28:MEC_v28',
max_episode_steps=5000,
)
register(
id='MECS-v29',
entry_point='sac.envs.env_V_sweep_v29:MEC_v29',
max_episode_steps=5000,
)
register(
id='MECS-v30',
entry_point='sac.envs.env_V_sweep_v30:MEC_v30',
max_episode_steps=5000,
)
env = normalize(ENVIRONMENTS[domain][task](**env_params))
pool = SimpleReplayBuffer(env_spec=env.spec, **replay_buffer_params)
sampler = SimpleSampler(**sampler_params)
base_kwargs = dict(algorithm_params['base_kwargs'], sampler=sampler)
M = value_fn_params['layer_size']
qf1 = NNQFunction(env_spec=env.spec, hidden_layer_sizes=(M, M), name='qf1')
qf2 = NNQFunction(env_spec=env.spec, hidden_layer_sizes=(M, M), name='qf2')
vf = NNVFunction(env_spec=env.spec, hidden_layer_sizes=(M, M))
initial_exploration_policy = UniformPolicy(env_spec=env.spec)
if policy_params['type'] == 'gaussian':
policy = GaussianPolicy(
env_spec=env.spec,
hidden_layer_sizes=(M,M),
reparameterize=policy_params['reparameterize'],
reg=1e-3,
)
elif policy_params['type'] == 'lsp':
nonlinearity = {
None: None,
'relu': tf.nn.relu,
'tanh': tf.nn.tanh
}[policy_params['preprocessing_output_nonlinearity']]
preprocessing_hidden_sizes = policy_params.get('preprocessing_hidden_sizes')
if preprocessing_hidden_sizes is not None:
observations_preprocessor = MLPPreprocessor(
env_spec=env.spec,
layer_sizes=preprocessing_hidden_sizes,
output_nonlinearity=nonlinearity)
else:
observations_preprocessor = None
policy_s_t_layers = policy_params['s_t_layers']
policy_s_t_units = policy_params['s_t_units']
s_t_hidden_sizes = [policy_s_t_units] * policy_s_t_layers
bijector_config = {
'num_coupling_layers': policy_params['coupling_layers'],
'translation_hidden_sizes': s_t_hidden_sizes,
'scale_hidden_sizes': s_t_hidden_sizes,
}
policy = LatentSpacePolicy(
env_spec=env.spec,
squash=policy_params['squash'],
bijector_config=bijector_config,
reparameterize=policy_params['reparameterize'],
q_function=qf1,
observations_preprocessor=observations_preprocessor)
elif policy_params['type'] == 'gmm':
# reparameterize should always be False if using a GMMPolicy
policy = GMMPolicy(
env_spec=env.spec,
K=policy_params['K'],
hidden_layer_sizes=(M, M),
reparameterize=policy_params['reparameterize'],
qf=qf1,
reg=1e-3,
)
else:
raise NotImplementedError(policy_params['type'])
algorithm = SAC(
base_kwargs=base_kwargs,
env=env,
policy=policy,
initial_exploration_policy=initial_exploration_policy,
pool=pool,
qf1=qf1,
qf2=qf2,
vf=vf,
lr=algorithm_params['lr'],
scale_reward=algorithm_params['scale']*algorithm_params['scale_reward'],
discount=algorithm_params['discount'],
tau=algorithm_params['tau'],
reparameterize=algorithm_params['reparameterize'],
target_update_interval=algorithm_params['target_update_interval'],
action_prior=policy_params['action_prior'],
save_full_state=False,
)
algorithm._sess.run(tf.global_variables_initializer())
algorithm.train()
def run(variant):
env = normalize(
MultiGoalEnv(
actuation_cost_coeff=1,
distance_cost_coeff=0.1,
goal_reward=1,
init_sigma=0.1,
))
pool = SimpleReplayBuffer(max_replay_buffer_size=1e6, env_spec=env.spec)
sampler = SimpleSampler(max_path_length=30,
min_pool_size=100,
batch_size=64)
base_kwargs = dict(sampler=sampler,
epoch_length=1000,
n_epochs=1000,
n_train_repeat=1,
eval_render=True,
eval_n_episodes=10,
eval_deterministic=False)
M = 128
qf = NNQFunction(env_spec=env.spec, hidden_layer_sizes=[M, M])
vf = NNVFunction(env_spec=env.spec, hidden_layer_sizes=[M, M])
if variant['policy_type'] == 'gmm':
policy = GMMPolicy(env_spec=env.spec,
K=4,
hidden_layer_sizes=[M, M],
qf=qf,
reg=0.001)
elif variant['policy_type'] == 'lsp':
bijector_config = {
"scale_regularization": 0.0,
"num_coupling_layers": 2,
"translation_hidden_sizes": (M, ),
"scale_hidden_sizes": (M, ),
}
policy = LatentSpacePolicy(env_spec=env.spec,
mode="train",
squash=True,
bijector_config=bijector_config,
observations_preprocessor=None)
plotter = QFPolicyPlotter(qf=qf,
policy=policy,
obs_lst=np.array([[-2.5, 0.0], [0.0, 0.0],
[2.5, 2.5]]),
default_action=[np.nan, np.nan],
n_samples=100)
algorithm = SAC(base_kwargs=base_kwargs,
env=env,
policy=policy,
pool=pool,
qf=qf,
vf=vf,
plotter=plotter,
lr=3e-4,
scale_reward=3.0,
discount=0.99,
tau=1e-4,
save_full_state=True)
algorithm.train()
def main(root_dir, seed, entropy_coeff, n_epochs, dynamic_coeff, clip_norm, regularize):
tf.set_random_seed(seed=seed)
env = GymEnv('MountainCarContinuous-v0')
env.min_action = env.action_space.low[0]
env.max_action = env.action_space.high[0]
env.env.seed(seed)
max_replay_buffer_size = int(1e6)
sampler_params = {'max_path_length': 1000, 'min_pool_size': 1000, 'batch_size': 128}
sampler = SimpleSampler(**sampler_params)
entropy_coeff = entropy_coeff
dynamic_coeff = dynamic_coeff
# env_id = 'ContinuousSpaceMaze{}_{}_RB{}_entropy_{}__Normalize'.format(goal[0], goal[1], max_replay_buffer_size, entropy_coeff)
env_id = 'MountainCarContinuous_RB1e6_entropy{}_epoch{}__Normalize_uniform'.format(entropy_coeff, n_epochs)
env_id = env_id + '_dynamicCoeff' if dynamic_coeff else env_id
os.makedirs(root_dir, exist_ok=True)
env_dir = os.path.join(root_dir, env_id)
os.makedirs(env_dir, exist_ok=True)
current_log_dir = os.path.join(env_dir, 'seed{}'.format(seed))
mylogger.make_log_dir(current_log_dir)
# env_id = 'Test'
print(env_id)
print('environment set done')
# define value function
layer_size = 100
qf = NNQFunction(env_spec=env.spec,
hidden_layer_sizes=(layer_size, layer_size))
vf = NNVFunction(env_spec=env.spec, hidden_layer_sizes=(layer_size, layer_size))
# use GMM policy
policy = GMMPolicy(
env_spec=env.spec,
K=4,
hidden_layer_sizes=[layer_size, layer_size],
qf=qf,
reg=1e-3,
squash=True
)
# TODO
base_kwargs = dict(
epoch_length=1000,
n_epochs=n_epochs,
# scale_reward=1,
n_train_repeat=1,
eval_render=False,
eval_n_episodes=20,
eval_deterministic=True,
)
pool = SimpleReplayBuffer(env_spec=env.spec, max_replay_buffer_size=max_replay_buffer_size)
base_kwargs = dict(base_kwargs, sampler=sampler)
algorithm = SAC(
base_kwargs=base_kwargs,
env=env,
policy=policy,
pool=pool,
qf=qf,
vf=vf,
lr=3e-4,
scale_reward=1.,
discount=0.99,
tau=1e-2,
target_update_interval=1,
action_prior='uniform',
save_full_state=False,
dynamic_coeff=dynamic_coeff,
entropy_coeff=entropy_coeff,
clip_norm=clip_norm,
)
# name = env_id + datetime.now().strftime("-%m%d-%Hh-%Mm-%ss")
# mylogger.make_log_dir(name)
algorithm._sess.run(tf.global_variables_initializer())
algorithm.train()
def main(env_id, seed, entropy_coeff, n_epochs, dynamic_coeff, clip_norm,
normalize_obs, buffer_size, max_path_length, min_pool_size,
batch_size, policy_mode, eval_model, e, stochastic):
tf.set_random_seed(seed=seed)
env = GymEnv(env_id)
env.min_action = env.action_space.low[0]
env.max_action = env.action_space.high[0]
if hasattr(env, "seed"):
env.seed(seed)
else:
env.env.seed(seed)
# define value function
layer_size = 100
qf = NNQFunction(env_spec=env.spec,
hidden_layer_sizes=(layer_size, layer_size))
vf = NNVFunction(env_spec=env.spec,
hidden_layer_sizes=(layer_size, layer_size))
print("here")
# use GMM policy
if policy_mode == "GMMPolicy":
# use GMM policy
policy = GMMPolicy(env_spec=env.spec,
K=4,
hidden_layer_sizes=[layer_size, layer_size],
qf=qf,
reg=1e-3,
squash=True)
elif policy_mode == "EExploitationPolicy":
policy = EExploitationPolicy(
env_spec=env.spec,
K=4,
hidden_layer_sizes=[layer_size, layer_size],
qf=qf,
reg=1e-3,
squash=True,
e=e)
else:
_, mode = str(policy_mode).split('-')
if _ != "Knack":
raise AssertionError(
"policy_mode should be GMMPolicy or Knack-p_control or Knack-exploitation or Knack-exploration"
)
else:
policy = KnackBasedPolicy(
a_lim_lows=env.action_space.low,
a_lim_highs=env.action_space.high,
mode=mode,
env_spec=env.spec,
K=4,
hidden_layer_sizes=[layer_size, layer_size],
qf=qf,
vf=vf,
reg=1e-3,
squash=True)
# TODO
base_kwargs = dict(
epoch_length=1000,
n_epochs=n_epochs,
# scale_reward=1,
n_train_repeat=1,
eval_render=False,
eval_n_episodes=20,
eval_deterministic=True,
)
max_replay_buffer_size = buffer_size
pool = SimpleReplayBuffer(env_spec=env.spec,
max_replay_buffer_size=max_replay_buffer_size)
sampler_params = {
'max_path_length': max_path_length,
'min_pool_size': min_pool_size,
'batch_size': batch_size
}
sampler = NormalizeSampler(
**sampler_params) if normalize_obs else SimpleSampler(**sampler_params)
base_kwargs = dict(base_kwargs, sampler=sampler)
algorithm = SAC(base_kwargs=base_kwargs,
env=env,
policy=policy,
pool=pool,
qf=qf,
vf=vf,
lr=3e-4,
scale_reward=1.,
discount=0.99,
tau=1e-2,
target_update_interval=1,
action_prior='uniform',
save_full_state=False,
dynamic_coeff=dynamic_coeff,
entropy_coeff=entropy_coeff,
clip_norm=clip_norm)
algorithm._sess.run(tf.global_variables_initializer())
# -------------- setting done ------------------------
# -------------- main process ------------------------
with algorithm._sess.as_default():
algorithm._saver.restore(algorithm._sess, eval_model)
if stochastic:
knack_file = os.path.join(os.path.dirname(eval_model),
"array/epoch0_2001.npz")
final_knacks = np.load(knack_file)['knack_kurtosis'][-1]
env = algorithm._env
if hasattr(env, "env"):
env = env.env
# np.random.seed(seed)
# env.seed(seed)
num_data = 50 # num_data * nprocess == 1500
steps_thresh = 1000
data = {'acs': [], 'ep_rets': [], 'obs': [], 'rews': []}
for i in range(num_data):
obs = env.reset()
done = False
steps = 0
ret = 0
tmp_data = {'acs': [], 'obs': [], 'rews': []}
if stochastic:
_min = np.min(final_knacks)
_max = np.max(final_knacks)
print("start episode {}".format(i))
while not done:
steps += 1
# env.render()
if stochastic:
if hasattr(algorithm.pi, "knack_thresh"):
v, mean, var, kurtosis = algorithm._policy.calc_and_update_knack(
[obs])
knack_value = kurtosis[0]
# _min = min(knack_value, _min)
# _max = max(knack_value, _max)
knack_value = (knack_value - _min) / (_max - _min)
if knack_value > 0.8: ## TODO hyper param
print("knack {}".format(knack_value))
was = algorithm._policy._is_deterministic
algorithm._policy._is_deterministic = True
action, _ = algorithm.policy.get_action(
obs.flatten())
algorithm._policy._is_deterministic = was
else:
action, _ = algorithm.policy.get_action(
obs.flatten())
else:
algorithm._policy._is_deterministic = False
action, _ = algorithm.policy.get_action(obs.flatten())
else:
if hasattr(algorithm._policy, "_is_deterministic"):
algorithm._policy._is_deterministic = True
action, _ = algorithm.policy.get_action(obs.flatten())
obs_next, rew, done, _ = env.step(action)
tmp_data['obs'].append(obs)
tmp_data['acs'].append(action)
tmp_data['rews'].append(rew)
ret += rew
obs = obs_next
if steps >= steps_thresh:
done = True
data['ep_rets'].append(ret)
for k, v in tmp_data.items():
data[k].append(v)
# np.savez_compressed("a.npz", **data)
# print("return mean: {}".format(np.mean(data['ep_rets'])))
return data
def main(env, seed, entropy_coeff, n_epochs, dynamic_coeff, clip_norm,
normalize_obs, buffer_size, max_path_length, min_pool_size,
batch_size, policy_mode):
tf.set_random_seed(seed=seed)
# define value function
layer_size = 100
qf = NNQFunction(env_spec=env.spec,
hidden_layer_sizes=(layer_size, layer_size))
vf = NNVFunction(env_spec=env.spec,
hidden_layer_sizes=(layer_size, layer_size))
if policy_mode == GMMPolicy:
# use GMM policy
policy = GMMPolicy(env_spec=env.spec,
K=4,
hidden_layer_sizes=[layer_size, layer_size],
qf=qf,
reg=1e-3,
squash=True)
else:
_, mode = str(policy_mode).split('-')
if _ != "Knack":
raise AssertionError(
"policy_mode should be GMMPolicy or Knack-p_control or Knack-exploitation or Knack-exploration"
)
else:
policy = KnackBasedPolicy(
a_lim_lows=env.action_space.low,
a_lim_highs=env.action_space.high,
mode=mode,
env_spec=env.spec,
K=4,
hidden_layer_sizes=[layer_size, layer_size],
qf=qf,
reg=1e-3,
squash=True)
# TODO
base_kwargs = dict(
epoch_length=1000,
n_epochs=n_epochs,
# scale_reward=1,
n_train_repeat=1,
eval_render=False,
eval_n_episodes=20,
eval_deterministic=True,
)
max_replay_buffer_size = buffer_size
pool = SimpleReplayBuffer(env_spec=env.spec,
max_replay_buffer_size=max_replay_buffer_size)
sampler_params = {
'max_path_length': max_path_length,
'min_pool_size': min_pool_size,
'batch_size': batch_size
}
sampler = NormalizeSampler(
**sampler_params) if normalize_obs else SimpleSampler(**sampler_params)
base_kwargs = dict(base_kwargs, sampler=sampler)
algorithm = SAC(base_kwargs=base_kwargs,
env=env,
policy=policy,
pool=pool,
qf=qf,
vf=vf,
lr=3e-4,
scale_reward=1.,
discount=0.99,
tau=1e-2,
target_update_interval=1,
action_prior='uniform',
save_full_state=False,
dynamic_coeff=dynamic_coeff,
entropy_coeff=entropy_coeff,
clip_norm=clip_norm)
algorithm._sess.run(tf.global_variables_initializer())
algorithm.train()
def run_experiment(variant):
env_params = variant['env_params']
policy_params = variant['policy_params']
value_fn_params = variant['value_fn_params']
algorithm_params = variant['algorithm_params']
replay_buffer_params = variant['replay_buffer_params']
sampler_params = variant['sampler_params']
task = variant['task']
domain = variant['domain']
env = normalize(ENVIRONMENTS[domain][task](**env_params))
pool = SimpleReplayBuffer(env_spec=env.spec, **replay_buffer_params)
sampler = SimpleSampler(**sampler_params)
base_kwargs = dict(algorithm_params['base_kwargs'], sampler=sampler)
M = value_fn_params['layer_size']
if variant['num_hidden'] != 256:
M = variant['num_hidden']
qf1 = NNQFunction(env_spec=env.spec,
hidden_layer_sizes=(M, M),
name='qf1',
batchnormvf=variant['batchnormvf'])
qf2 = NNQFunction(env_spec=env.spec,
hidden_layer_sizes=(M, M),
name='qf2',
batchnormvf=variant['batchnormvf'])
vf = NNVFunction(env_spec=env.spec,
hidden_layer_sizes=(M, M),
batchnormvf=variant['batchnormvf'],
dropoutvf_keep_prob=variant['dropoutvf'])
initial_exploration_policy = UniformPolicy(env_spec=env.spec)
if policy_params['type'] == 'gaussian':
policy = GaussianPolicy(env_spec=env.spec,
hidden_layer_sizes=(M, M),
reparameterize=policy_params['reparameterize'],
todropoutpi=(variant['dropoutpi'] < 1.0),
dropoutpi=variant['dropoutpi'],
batchnormpi=variant['batchnormpi'])
elif policy_params['type'] == 'lsp':
nonlinearity = {
None: None,
'relu': tf.nn.relu,
'tanh': tf.nn.tanh
}[policy_params['preprocessing_output_nonlinearity']]
preprocessing_hidden_sizes = policy_params.get(
'preprocessing_hidden_sizes')
if preprocessing_hidden_sizes is not None:
observations_preprocessor = MLPPreprocessor(
env_spec=env.spec,
layer_sizes=preprocessing_hidden_sizes,
output_nonlinearity=nonlinearity)
else:
observations_preprocessor = None
policy_s_t_layers = policy_params['s_t_layers']
policy_s_t_units = policy_params['s_t_units']
s_t_hidden_sizes = [policy_s_t_units] * policy_s_t_layers
bijector_config = {
'num_coupling_layers': policy_params['coupling_layers'],
'translation_hidden_sizes': s_t_hidden_sizes,
'scale_hidden_sizes': s_t_hidden_sizes,
}
policy = LatentSpacePolicy(
env_spec=env.spec,
squash=policy_params['squash'],
bijector_config=bijector_config,
reparameterize=policy_params['reparameterize'],
q_function=qf1,
observations_preprocessor=observations_preprocessor)
elif policy_params['type'] == 'gmm':
# reparameterize should always be False if using a GMMPolicy
policy = GMMPolicy(
env_spec=env.spec,
K=policy_params['K'],
hidden_layer_sizes=(M, M),
reparameterize=policy_params['reparameterize'],
qf=qf1,
reg=1e-3,
)
else:
raise NotImplementedError(policy_params['type'])
if variant['reward_scale'] < 0:
scale_rew = algorithm_params['scale_reward']
else:
scale_rew = variant['reward_scale']
algorithm = SAC(
base_kwargs=base_kwargs,
env=env,
policy=policy,
initial_exploration_policy=initial_exploration_policy,
pool=pool,
qf1=qf1,
qf2=qf2,
vf=vf,
lr=algorithm_params['lr'] if variant['lr'] < 0 else variant['lr'],
scale_reward=scale_rew,
discount=algorithm_params['discount'],
tau=variant['tau'],
reparameterize=algorithm_params['reparameterize'],
target_update_interval=algorithm_params['target_update_interval'],
action_prior=policy_params['action_prior'],
save_full_state=False,
l1regpi=variant['l1regpi'],
l2regpi=variant['l2regpi'],
l1regvf=variant['l1regvf'],
l2regvf=variant['l2regvf'],
ent_coef=variant['ent_coef'],
wclippi=variant['wclippi'],
wclipvf=variant['wclipvf'],
dropoutpi=variant['dropoutpi'],
dropoutvf=variant['dropoutvf'],
batchnormpi=variant['batchnormpi'],
batchnormvf=variant['batchnormvf'])
algorithm._sess.run(tf.global_variables_initializer())
for v in tf.trainable_variables():
print(v.name)
algorithm.train()
if variant['policypath'] != '':
save_w_path = os.path.expanduser(variant['policypath'])
toexport = []
savesess = algorithm._sess
for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='gaussian_policy'):
toexport.append(savesess.run(v))
np.savetxt(save_w_path,
np.concatenate(toexport, axis=None),
delimiter=',')
if variant['valuepath'] != '':
save_w_path = os.path.expanduser(variant['valuepath'])
toexport = []
savesess = algorithm._sess
for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='qf1'):
toexport.append(savesess.run(v))
for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='qf2'):
toexport.append(savesess.run(v))
for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='vf'):
toexport.append(savesess.run(v))
np.savetxt(save_w_path,
np.concatenate(toexport, axis=None),
delimiter=',')
def main(root_dir):
# tf.set_random_seed(seed=seed)
# env = GymEnv('MountainCarContinuous-v0')
env = GymEnv('MountainCarContinuousColor-v0')
max_replay_buffer_size = int(1e6)
sampler_params = {'max_path_length': 1000, 'min_pool_size': 1000, 'batch_size': 128}
# TODO Normalize or not
sampler = SimpleSampler(**sampler_params)
entropy_coeff = 0.
dynamic_coeff = True
# define value function
layer_size = 100
qf = NNQFunction(env_spec=env.spec, hidden_layer_sizes=(layer_size, layer_size))
vf = NNVFunction(env_spec=env.spec, hidden_layer_sizes=(layer_size, layer_size))
# use GMM policy
policy = GMMPolicy(
env_spec=env.spec,
K=4,
hidden_layer_sizes=[layer_size, layer_size],
qf=qf,
reg=1e-3,
squash=True
)
# TODO
base_kwargs = dict(
epoch_length=1000,
n_epochs=10,
# scale_reward=1,
n_train_repeat=1,
eval_render=False,
eval_n_episodes=20,
eval_deterministic=True,
)
pool = SimpleReplayBuffer(env_spec=env.spec, max_replay_buffer_size=max_replay_buffer_size)
base_kwargs = dict(base_kwargs, sampler=sampler)
algorithm = SAC(
base_kwargs=base_kwargs,
env=env,
policy=policy,
pool=pool,
qf=qf,
vf=vf,
lr=3e-4,
scale_reward=1.,
discount=0.99,
tau=1e-2,
target_update_interval=1,
action_prior='uniform',
save_full_state=False,
dynamic_coeff=dynamic_coeff,
entropy_coeff=entropy_coeff
)
algorithm._sess.run(tf.global_variables_initializer())
# TODO Normalize or not
# Currently only MountainCar is available
with algorithm._sess.as_default():
model_file = os.path.join(root_dir, 'model')
algorithm._saver.restore(algorithm._sess, model_file)
for i in range(1):
obs = env.reset()
env.env.render()
sleep(4.0)
traj = [obs]
done = False
while not done:
env.env.render()
action = algorithm.policy.get_action(obs.flatten())
obs, rew, done, _ = env.step(action)
traj.append(obs.flatten())
knack, knack_kurtosis = sub_goal_detect(algorithm, traj)
idxs = np.argsort(knack_kurtosis)
# idxs = np.argsort(knack)
print(idxs[::-1])
COL = MplColorHelper('Blues', np.min(knack_kurtosis), np.max(knack_kurtosis))
for j, s in enumerate(traj):
env.env.state = np.array(traj[j])
rgba = COL.get_rgb(knack_kurtosis[j])
env.env.render(car_rgba=rgba)
sleep(1.0)
for idx in idxs[::-1]:
obs = env.reset()
env.env.state = np.array(traj[0])
rgba = COL.get_rgb(knack_kurtosis[0])
env.env.render(car_rgba=rgba)
for j in range(idx+1):
env.env.state = np.array(traj[j])
rgba = COL.get_rgb(knack_kurtosis[j])
# env.env.viewer.geoms[1].set_color(*(0.0, 0.0, 1.0))
env.env.render(car_rgba=rgba)
sleep(0.5)
