def main():
n_seeds = 1
mode = "here"
exp_prefix = "dev-sl"
# n_seeds = 10
# mode = "ec2"
exp_prefix = "paper-6-14-HL-sl-H25"
H = 25
# noinspection PyTypeChecker
variant = dict(
H=H,
exp_prefix=exp_prefix,
algo_params=dict(
num_batches_per_epoch=100,
num_epochs=30,
learning_rate=1e-3,
batch_size=1000,
eval_num_episodes=64,
lstm_state_size=10,
# rnn_cell_class=LSTMCell,
# rnn_cell_params=dict(
# use_peepholes=True,
# ),
rnn_cell_class=SeparateLstmLinearCell,
rnn_cell_params=dict(
use_peepholes=True,
env_noise_std=0,
memory_noise_std=0,
output_nonlinearity=tf.nn.tanh,
# output_nonlinearity=tf.nn.softmax,
env_hidden_sizes=[],
output_dim=1,
),
softmax=False,
),
version='Supervised Learning',
env_class=HighLow,
env_params=dict(horizon=H, )
# env_class=OneCharMemory,
)
exp_id = -1
for _ in range(n_seeds):
seed = random.randint(0, 999999)
exp_id += 1
set_seed(seed)
variant['seed'] = seed
variant['exp_id'] = exp_id
run_experiment(
bptt_launcher,
exp_prefix=exp_prefix,
seed=seed,
mode=mode,
variant=variant,
exp_id=exp_id,
)
def run_linear_ocm_exp(variant):
from railrl.tf.ddpg import DDPG
from railrl.envs.flattened_product_box import FlattenedProductBox
from railrl.exploration_strategies.ou_strategy import OUStrategy
from railrl.tf.policies.nn_policy import FeedForwardPolicy
from railrl.qfunctions.nn_qfunction import FeedForwardCritic
from railrl.envs.memory.continuous_memory_augmented import (
ContinuousMemoryAugmented
)
from railrl.launchers.launcher_util import (
set_seed,
)
"""
Set up experiment variants.
"""
seed = variant['seed']
algo_params = variant['algo_params']
env_class = variant['env_class']
env_params = variant['env_params']
memory_dim = variant['memory_dim']
ou_params = variant['ou_params']
set_seed(seed)
"""
Code for running the experiment.
"""
env = env_class(**env_params)
env = ContinuousMemoryAugmented(
env,
num_memory_states=memory_dim,
)
env = FlattenedProductBox(env)
qf = FeedForwardCritic(
name_or_scope="critic",
env_spec=env.spec,
)
policy = FeedForwardPolicy(
name_or_scope="policy",
env_spec=env.spec,
)
es = OUStrategy(
env_spec=env.spec,
**ou_params
)
algorithm = DDPG(
env,
es,
policy,
qf,
**algo_params
)
algorithm.train()
def run_linear_ocm_exp(variant):
from sandbox.rocky.tf.algos.trpo import TRPO
from rllab.baselines.linear_feature_baseline import LinearFeatureBaseline
from sandbox.rocky.tf.policies.gaussian_lstm_policy import GaussianLSTMPolicy
import sandbox.rocky.tf.core.layers as L
from sandbox.rocky.tf.optimizers.conjugate_gradient_optimizer import (
ConjugateGradientOptimizer,
FiniteDifferenceHvp,
)
from railrl.envs.flattened_product_box import FlattenedProductBox
from railrl.envs.memory.continuous_memory_augmented import (
ContinuousMemoryAugmented)
from railrl.envs.memory.one_char_memory import (
OneCharMemoryEndOnly, )
from railrl.envs.memory.high_low import HighLow
from railrl.launchers.launcher_util import (
set_seed, )
"""
Set up experiment variants.
"""
H = variant['H']
seed = variant['seed']
num_values = variant['num_values']
set_seed(seed)
onehot_dim = num_values + 1
"""
Code for running the experiment.
"""
# env = OneCharMemoryEndOnly(n=num_values, num_steps=H, softmax_action=True)
env = HighLow(num_steps=H)
env = ContinuousMemoryAugmented(
env,
num_memory_states=onehot_dim,
)
env = FlattenedProductBox(env)
policy = GaussianLSTMPolicy(
name="policy",
env_spec=env.spec,
lstm_layer_cls=L.LSTMLayer,
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
optimizer_params = variant['optimizer_params']
trpo_params = variant['trpo_params']
algo = TRPO(env=env,
policy=policy,
baseline=baseline,
optimizer=ConjugateGradientOptimizer(
hvp_approach=FiniteDifferenceHvp(**optimizer_params)),
**trpo_params)
algo.train()
def run_linear_ocm_exp(variant):
from railrl.tf.ddpg import DDPG
from railrl.envs.memory.continuous_memory_augmented import (
ContinuousMemoryAugmented)
from railrl.envs.memory.one_char_memory import (
OneCharMemoryEndOnly, )
from railrl.launchers.launcher_util import (
set_seed, )
"""
Set up experiment variants.
"""
H = variant['H']
seed = variant['seed']
num_values = variant['num_values']
algo_params = variant['algo_params']
set_seed(seed)
onehot_dim = num_values + 1
env_action_dim = num_values + 1
"""
Code for running the experiment.
"""
env = OneCharMemoryEndOnly(n=num_values, num_steps=H, softmax_action=True)
env = ContinuousMemoryAugmented(
env,
num_memory_states=onehot_dim,
)
# env = FlattenedProductBox(env)
# qf = FeedForwardCritic(
# name_or_scope="critic",
# env_spec=env.spec,
# )
qf = MlpMemoryQFunction(
name_or_scope="critic",
env_spec=env.spec,
)
policy = ActionAwareMemoryPolicy(
name_or_scope="noisy_policy",
action_dim=env_action_dim,
memory_dim=memory_dim,
env_spec=env.spec,
)
es = OUStrategy(env_spec=env.spec)
algorithm = DDPG(env, es, policy, qf, **algo_params)
algorithm.train()
def experiment(variant):
from railrl.torch.ddpg import DDPG
from railrl.launchers.launcher_util import (
set_seed, )
seed = variant['seed']
algo_params = variant['algo_params']
env_params = variant['env_params']
es_class = variant['es_class']
es_params = variant['es_params']
set_seed(seed)
env = TwoDPoint(**env_params)
es = es_class(env_spec=env.spec, **es_params)
algorithm = DDPG(env, es, **algo_params)
algorithm.train()
def run_linear_ocm_exp(variant):
from railrl.tf.ddpg_ocm import DdpgOcm
from railrl.qfunctions.memory.mlp_memory_qfunction import MlpMemoryQFunction
from railrl.exploration_strategies.noop import NoopStrategy
from railrl.exploration_strategies.onehot_sampler import OneHotSampler
from railrl.exploration_strategies.product_strategy import ProductStrategy
from railrl.envs.memory.continuous_memory_augmented import (
ContinuousMemoryAugmented)
from railrl.envs.memory.one_char_memory import OneCharMemoryEndOnly
from railrl.tf.policies.memory.linear_ocm_policy import LinearOcmPolicy
from railrl.launchers.launcher_util import (
set_seed, )
"""
Set up experiment variants.
"""
H = variant['H']
seed = variant['seed']
num_values = variant['num_values']
ddpg_params = variant['ddpg_params']
onehot_dim = num_values + 1
set_seed(seed)
"""
Code for running the experiment.
"""
env = OneCharMemoryEndOnly(n=num_values, num_steps=H)
env = ContinuousMemoryAugmented(
env,
num_memory_states=onehot_dim,
)
policy = LinearOcmPolicy(
name_or_scope="policy",
memory_and_action_dim=onehot_dim,
env_spec=env.spec,
)
es = ProductStrategy([OneHotSampler(), NoopStrategy()])
qf = MlpMemoryQFunction(
name_or_scope="critic",
env_spec=env.spec,
)
algorithm = DdpgOcm(env, es, policy, qf, **ddpg_params)
algorithm.train()
def run_linear_ocm_exp(variant):
from railrl.tf.ddpg import DDPG
from railrl.launchers.launcher_util import (
set_seed, )
from railrl.exploration_strategies.ou_strategy import OUStrategy
from railrl.tf.policies.nn_policy import FeedForwardPolicy
from railrl.qfunctions.nn_qfunction import FeedForwardCritic
"""
Set up experiment variants.
"""
H = variant['H']
seed = variant['seed']
algo_params = variant['algo_params']
env_class = variant['env_class']
env_params = variant['env_params']
ou_params = variant['ou_params']
set_seed(seed)
"""
Code for running the experiment.
"""
env = env_class(**env_params)
qf = FeedForwardCritic(
name_or_scope="critic",
env_spec=env.spec,
)
policy = FeedForwardPolicy(
name_or_scope="policy",
env_spec=env.spec,
)
es = OUStrategy(env_spec=env.spec, **ou_params)
algorithm = DDPG(env, es, policy, qf, **algo_params)
algorithm.train()
for launcher in [
# trpo_launcher,
# mem_trpo_launcher,
# rtrpo_launcher,
ddpg_launcher,
mem_ddpg_launcher,
rdpg_launcher,
]:
search_space = {
# 'env_class': [WaterMaze1D, WaterMazeEasy1D, WaterMazeMemory1D],
}
sweeper = DeterministicHyperparameterSweeper(
search_space, default_parameters=variant)
for exp_id, variant in enumerate(sweeper.iterate_hyperparameters()):
for seed in range(n_seeds):
exp_id += 1
set_seed(seed)
variant['seed'] = seed
variant['exp_id'] = exp_id
run_experiment(
launcher,
exp_prefix=exp_prefix,
seed=seed,
mode=mode,
variant=variant,
exp_id=exp_id,
snapshot_mode='last',
use_gpu=use_gpu,
)
def main():
n_seeds = 1
mode = "here"
exp_prefix = "dev-sl"
# n_seeds = 10
# mode = "ec2"
# exp_prefix = "6-2-sl-rwa-vs-lstm"
env_noise_std = 0
memory_noise_std = 0
for rnn_cell_class, H in product(
[SeparateRWALinearCell],
[512],
# [RWACell, LSTMCell, GRUCell],
# [512, 256, 128, 64],
):
# noinspection PyTypeChecker
variant = dict(
H=H,
exp_prefix=exp_prefix,
algo_params=dict(
num_batches_per_epoch=10000 // 32,
num_epochs=100,
learning_rate=1e-3,
batch_size=32,
eval_num_episodes=64,
lstm_state_size=10,
rnn_cell_class=rnn_cell_class,
rnn_cell_params=dict(
# use_peepholes=True,
state_is_flat_externally=False,
output_dim=1,
),
# rnn_cell_class=SeparateLstmLinearCell,
# rnn_cell_params=dict(
# use_peepholes=True,
# env_noise_std=env_noise_std,
# memory_noise_std=memory_noise_std,
# output_nonlinearity=tf.nn.tanh,
# # output_nonlinearity=tf.nn.softmax,
# env_hidden_sizes=[],
# ),
softmax=False,
),
version='Supervised Learning',
env_class=HighLow,
# env_class=OneCharMemory,
)
exp_id = -1
for seed in range(n_seeds):
exp_id += 1
set_seed(seed)
variant['seed'] = seed
variant['exp_id'] = exp_id
run_experiment(
bptt_launcher,
exp_prefix=exp_prefix,
seed=seed,
mode=mode,
variant=variant,
exp_id=exp_id,
)