def run_gym(
params,
offline_train,
score_bar,
gpu_id,
save_timesteps_to_dataset=None,
start_saving_from_score=None,
path_to_pickled_transitions=None,
):
logger.info("Running gym with params")
logger.info(params)
rl_parameters = RLParameters(**params["rl"])
env_type = params["env"]
model_type = params["model_type"]
epsilon, epsilon_decay, minimum_epsilon = create_epsilon(
offline_train, rl_parameters, params
)
env = OpenAIGymEnvironment(
env_type,
epsilon,
rl_parameters.softmax_policy,
rl_parameters.gamma,
epsilon_decay,
minimum_epsilon,
)
replay_buffer = create_replay_buffer(
env, params, model_type, offline_train, path_to_pickled_transitions
)
use_gpu = gpu_id != USE_CPU
trainer = create_trainer(params["model_type"], params, rl_parameters, use_gpu, env)
predictor = create_predictor(trainer, model_type, use_gpu, env.action_dim)
c2_device = core.DeviceOption(
caffe2_pb2.CUDA if use_gpu else caffe2_pb2.CPU, int(gpu_id)
)
return train(
c2_device,
env,
offline_train,
replay_buffer,
model_type,
trainer,
predictor,
"{} test run".format(env_type),
score_bar,
**params["run_details"],
save_timesteps_to_dataset=save_timesteps_to_dataset,
start_saving_from_score=start_saving_from_score,
)
def run_gym(
params,
use_gpu,
score_bar,
embed_rl_dataset: RLDataset,
gym_env: Env,
mdnrnn: MemoryNetwork,
max_embed_seq_len: int,
):
rl_parameters = RLParameters(**params["rl"])
env_type = params["env"]
model_type = params["model_type"]
epsilon, epsilon_decay, minimum_epsilon = create_epsilon(
offline_train=True, rl_parameters=rl_parameters, params=params)
replay_buffer = OpenAIGymMemoryPool(params["max_replay_memory_size"])
for row in embed_rl_dataset.rows:
replay_buffer.insert_into_memory(**row)
state_mem = np.array([m[0] for m in replay_buffer.replay_memory])
state_min_value = np.amin(state_mem)
state_max_value = np.amax(state_mem)
state_embed_env = StateEmbedGymEnvironment(gym_env, mdnrnn,
max_embed_seq_len,
state_min_value,
state_max_value)
open_ai_env = OpenAIGymEnvironment(
state_embed_env,
epsilon,
rl_parameters.softmax_policy,
rl_parameters.gamma,
epsilon_decay,
minimum_epsilon,
)
rl_trainer = create_trainer(params["model_type"], params, rl_parameters,
use_gpu, open_ai_env)
rl_predictor = create_predictor(rl_trainer, model_type, use_gpu,
open_ai_env.action_dim)
return train_gym_offline_rl(
open_ai_env,
replay_buffer,
model_type,
rl_trainer,
rl_predictor,
"{} offline rl state embed".format(env_type),
score_bar,
max_steps=params["run_details"]["max_steps"],
avg_over_num_episodes=params["run_details"]["avg_over_num_episodes"],
offline_train_epochs=params["run_details"]["offline_train_epochs"],
bcq_imitator_hyper_params=None,
)
def test_minibatches_per_step(self):
_epochs = self.epochs
self.epochs = 2
rl_parameters = RLParameters(gamma=0.95,
target_update_rate=0.9,
maxq_learning=True)
rainbow_parameters = RainbowDQNParameters(double_q_learning=True,
dueling_architecture=False)
training_parameters1 = TrainingParameters(
layers=self.layers,
activations=self.activations,
minibatch_size=1024,
minibatches_per_step=1,
learning_rate=0.25,
optimizer="ADAM",
)
training_parameters2 = TrainingParameters(
layers=self.layers,
activations=self.activations,
minibatch_size=128,
minibatches_per_step=8,
learning_rate=0.25,
optimizer="ADAM",
)
env1 = Env(self.state_dims, self.action_dims)
env2 = Env(self.state_dims, self.action_dims)
model_parameters1 = DiscreteActionModelParameters(
actions=env1.actions,
rl=rl_parameters,
rainbow=rainbow_parameters,
training=training_parameters1,
)
model_parameters2 = DiscreteActionModelParameters(
actions=env2.actions,
rl=rl_parameters,
rainbow=rainbow_parameters,
training=training_parameters2,
)
# minibatch_size / 8, minibatches_per_step * 8 should give the same result
logger.info("Training model 1")
trainer1 = self._train(model_parameters1, env1)
SummaryWriterContext._reset_globals()
logger.info("Training model 2")
trainer2 = self._train(model_parameters2, env2)
weight1 = trainer1.q_network.fc.layers[-1].weight.detach().numpy()
weight2 = trainer2.q_network.fc.layers[-1].weight.detach().numpy()
# Due to numerical stability this tolerance has to be fairly high
self.assertTrue(np.allclose(weight1, weight2, rtol=0.0, atol=1e-3))
self.epochs = _epochs
def test_trainer_maxq(self):
environment = Gridworld()
maxq_sarsa_parameters = DiscreteActionModelParameters(
actions=environment.ACTIONS,
rl=RLParameters(
gamma=DISCOUNT,
target_update_rate=0.5,
reward_burnin=10,
maxq_learning=True,
),
training=TrainingParameters(
layers=[-1, 1],
activations=["linear"],
minibatch_size=self.minibatch_size,
learning_rate=0.01,
optimizer="ADAM",
),
)
# construct the new trainer that using maxq
maxq_trainer = DiscreteActionTrainer(
maxq_sarsa_parameters, environment.normalization
)
samples = environment.generate_samples(100000, 1.0)
predictor = maxq_trainer.predictor()
tdps = environment.preprocess_samples(samples, self.minibatch_size)
evaluator = GridworldEvaluator(environment, True)
evaluator.evaluate(predictor)
print(
"Pre-Training eval: ",
evaluator.mc_loss[-1],
evaluator.reward_doubly_robust[-1],
)
self.assertGreater(evaluator.mc_loss[-1], 0.3)
for _ in range(5):
for tdp in tdps:
maxq_trainer.train_numpy(tdp, None)
evaluator.evaluate(predictor)
print(
"Post-Training eval: ",
evaluator.mc_loss[-1],
evaluator.reward_doubly_robust[-1],
)
self.assertLess(evaluator.mc_loss[-1], 0.1)
self.assertGreater(
evaluator.reward_doubly_robust[-1], evaluator.reward_doubly_robust[-2]
)
def test_trainer_maxq(self):
env = Env(self.state_dims, self.action_dims)
env.seed(42)
maxq_parameters = DiscreteActionModelParameters(
actions=env.actions,
rl=RLParameters(
gamma=0.99,
target_update_rate=0.5,
reward_burnin=10,
maxq_learning=True
),
training=TrainingParameters(
layers=self.layers,
activations=self.activations,
minibatch_size=self.minibatch_size,
learning_rate=0.01,
optimizer='ADAM',
)
)
maxq_trainer = DiscreteActionTrainer(
maxq_parameters,
env.normalization
)
# predictor = maxq_trainer.predictor()
logger.info('Generating constant_reward MDPs..')
states, actions, rewards, next_states, next_actions, is_terminal, \
possible_next_actions, reward_timelines = \
env.generate_samples_discrete(self.num_samples)
logger.info('Preprocessing constant_reward MDPs..')
tdps = env.preprocess_samples_discrete(states, actions, rewards, next_states,
next_actions, is_terminal, possible_next_actions, reward_timelines,
self.minibatch_size,)
for epoch in range(self.epochs):
logger.info('Training..', epoch)
for tdp in tdps:
maxq_trainer.train_numpy(tdp, None)
logger.info('Training epoch', epoch, 'average q values',
np.mean(workspace.FetchBlob(maxq_trainer.q_score_output)),
'td_loss', workspace.FetchBlob(maxq_trainer.loss_blob))
# Q value should converge to very close to 100
avg_q_value_after_training = np.mean(
workspace.FetchBlob(maxq_trainer.q_score_output))
self.assertLess(avg_q_value_after_training, 101)
self.assertGreater(avg_q_value_after_training, 99)
def main(params):
# Set minibatch size based on # of devices being used to train
params["shared_training"]["minibatch_size"] *= minibatch_size_multiplier(
params["use_gpu"], params["use_all_avail_gpus"])
rl_parameters = RLParameters(**params["rl"])
training_parameters = DDPGTrainingParameters(**params["shared_training"])
actor_parameters = DDPGNetworkParameters(**params["actor_training"])
critic_parameters = DDPGNetworkParameters(**params["critic_training"])
model_params = DDPGModelParameters(
rl=rl_parameters,
shared_training=training_parameters,
actor_training=actor_parameters,
critic_training=critic_parameters,
)
state_normalization = BaseWorkflow.read_norm_file(
params["state_norm_data_path"])
action_normalization = BaseWorkflow.read_norm_file(
params["action_norm_data_path"])
writer = SummaryWriter(log_dir=params["model_output_path"])
logger.info("TensorBoard logging location is: {}".format(writer.log_dir))
preprocess_handler = ContinuousPreprocessHandler(
Preprocessor(state_normalization, False),
Preprocessor(action_normalization, False),
PandasSparseToDenseProcessor(),
)
workflow = ContinuousWorkflow(
model_params,
preprocess_handler,
state_normalization,
action_normalization,
params["use_gpu"],
params["use_all_avail_gpus"],
)
train_dataset = JSONDatasetReader(
params["training_data_path"],
batch_size=training_parameters.minibatch_size)
eval_dataset = JSONDatasetReader(params["eval_data_path"], batch_size=16)
with summary_writer_context(writer):
workflow.train_network(train_dataset, eval_dataset,
int(params["epochs"]))
return export_trainer_and_predictor(workflow.trainer,
params["model_output_path"]) # noqa
def get_td3_parameters(self, use_2_q_functions=False):
return TD3ModelParameters(
rl=RLParameters(gamma=DISCOUNT, target_update_rate=0.01),
training=TD3TrainingParameters(
minibatch_size=self.minibatch_size,
use_2_q_functions=use_2_q_functions,
q_network_optimizer=OptimizerParameters(),
actor_network_optimizer=OptimizerParameters(),
),
q_network=FeedForwardParameters(layers=[128, 64],
activations=["relu", "relu"]),
actor_network=FeedForwardParameters(layers=[128, 64],
activations=["relu", "relu"]),
)
def setUp(self):
super(self.__class__, self).setUp()
np.random.seed(0)
random.seed(0)
self.state_dim, self.action_dim = 2, 3
self._env = MockEnv(self.state_dim, self.action_dim)
self._rl_parameters = RLParameters(
gamma=0.9,
target_update_rate=0.5,
reward_burnin=10,
maxq_learning=False,
)
self._rl_parameters_maxq = RLParameters(
gamma=0.9,
target_update_rate=0.5,
reward_burnin=10,
maxq_learning=True,
)
self._rl_parameters = ContinuousActionModelParameters(
rl=self._rl_parameters,
training=TrainingParameters(
layers=[
-1, self._env.num_states * self._env.num_actions * 2, 1
],
activations=['linear', 'linear'],
minibatch_size=1024,
learning_rate=0.01,
optimizer='ADAM',
),
knn=KnnParameters(model_type='DQN', ))
self._trainer = ContinuousActionDQNTrainer(
self._env.normalization, self._env.normalization_action,
self._rl_parameters)
def get_sarsa_parameters(self):
return ContinuousActionModelParameters(
rl=RLParameters(gamma=DISCOUNT,
target_update_rate=1.0,
maxq_learning=False),
training=TrainingParameters(
layers=[-1, 256, 128, -1],
activations=["relu", "relu", "linear"],
minibatch_size=self.minibatch_size,
learning_rate=0.05,
optimizer="ADAM",
),
rainbow=RainbowDQNParameters(double_q_learning=True,
dueling_architecture=False),
)
def get_sarsa_parameters(self):
return ContinuousActionModelParameters(
rl=RLParameters(
gamma=DISCOUNT,
target_update_rate=0.5,
reward_burnin=10,
maxq_learning=False,
),
training=TrainingParameters(
layers=[-1, 200, 1],
activations=['linear', 'linear'],
minibatch_size=1024,
learning_rate=0.01,
optimizer='ADAM',
),
knn=KnnParameters(model_type='DQN', ))
def test_trainer_maxq(self):
environment = Gridworld()
maxq_sarsa_parameters = DiscreteActionModelParameters(
actions=environment.ACTIONS,
rl=RLParameters(gamma=DISCOUNT,
target_update_rate=0.5,
reward_burnin=10,
maxq_learning=True),
training=TrainingParameters(
layers=[-1, 1],
activations=['linear'],
minibatch_size=self.minibatch_size,
learning_rate=0.01,
optimizer='ADAM',
))
# construct the new trainer that using maxq
maxq_trainer = DiscreteActionTrainer(
maxq_sarsa_parameters,
environment.normalization,
)
states, actions, rewards, next_states, next_actions, is_terminal,\
possible_next_actions, reward_timelines = \
environment.generate_samples(100000, 1.0)
predictor = maxq_trainer.predictor()
tdps = environment.preprocess_samples(
states,
actions,
rewards,
next_states,
next_actions,
is_terminal,
possible_next_actions,
reward_timelines,
self.minibatch_size,
)
evaluator = GridworldEvaluator(environment, True)
print("Pre-Training eval", evaluator.evaluate(predictor))
self.assertGreater(evaluator.evaluate(predictor), 0.3)
for _ in range(2):
for tdp in tdps:
maxq_trainer.stream_tdp(tdp, None)
evaluator.evaluate(predictor)
print("Post-Training eval", evaluator.evaluate(predictor))
self.assertLess(evaluator.evaluate(predictor), 0.1)
def get_sarsa_trainer(self, environment):
rl_parameters = RLParameters(gamma=DISCOUNT,
target_update_rate=0.5,
reward_burnin=10,
maxq_learning=False)
training_parameters = TrainingParameters(
layers=[-1, 1],
activations=['linear'],
minibatch_size=1024,
learning_rate=0.01,
optimizer='ADAM',
)
return DiscreteActionTrainer(
environment.normalization,
DiscreteActionModelParameters(actions=environment.ACTIONS,
rl=rl_parameters,
training=training_parameters))
def run_gym(
params,
score_bar,
gpu_id,
save_timesteps_to_dataset=None,
start_saving_from_episode=0,
):
# Caffe2 core uses the min of caffe2_log_level and minloglevel
# to determine loglevel. See caffe2/caffe2/core/logging.cc for more info.
core.GlobalInit(["caffe2", "--caffe2_log_level=2", "--minloglevel=2"])
logger.info("Running gym with params")
logger.info(params)
rl_parameters = RLParameters(**params["rl"])
env_type = params["env"]
env = OpenAIGymEnvironment(
env_type,
rl_parameters.epsilon,
rl_parameters.softmax_policy,
rl_parameters.gamma,
)
replay_buffer = OpenAIGymMemoryPool(params["max_replay_memory_size"])
model_type = params["model_type"]
use_gpu = gpu_id != USE_CPU
trainer = create_trainer(params["model_type"], params, rl_parameters, use_gpu, env)
predictor = create_predictor(trainer, model_type, use_gpu)
c2_device = core.DeviceOption(
caffe2_pb2.CUDA if use_gpu else caffe2_pb2.CPU, gpu_id
)
return train_sgd(
c2_device,
env,
replay_buffer,
model_type,
trainer,
predictor,
"{} test run".format(env_type),
score_bar,
**params["run_details"],
save_timesteps_to_dataset=save_timesteps_to_dataset,
start_saving_from_episode=start_saving_from_episode,
)
def train_network(params):
logger.info("Running DQN workflow with params:")
logger.info(params)
action_names = np.array(params["actions"])
rl_parameters = RLParameters(**params["rl"])
training_parameters = TrainingParameters(**params["training"])
rainbow_parameters = RainbowDQNParameters(**params["rainbow"])
trainer_params = DiscreteActionModelParameters(
actions=params["actions"],
rl=rl_parameters,
training=training_parameters,
rainbow=rainbow_parameters,
)
dataset = JSONDataset(params["training_data_path"],
batch_size=training_parameters.minibatch_size)
norm_data = JSONDataset(params["state_norm_data_path"])
state_normalization = read_norm_params(norm_data.read_all())
num_batches = int(len(dataset) / training_parameters.minibatch_size)
logger.info("Read in batch data set {} of size {} examples. Data split "
"into {} batches of size {}.".format(
params["training_data_path"],
len(dataset),
num_batches,
training_parameters.minibatch_size,
))
trainer = DQNTrainer(trainer_params, state_normalization,
params["use_gpu"])
for epoch in range(params["epochs"]):
for batch_idx in range(num_batches):
helpers.report_training_status(batch_idx, num_batches, epoch,
params["epochs"])
batch = dataset.read_batch(batch_idx)
tdp = preprocess_batch_for_training(action_names, batch,
state_normalization)
trainer.train(tdp)
logger.info("Training finished. Saving PyTorch model to {}".format(
params["pytorch_output_path"]))
helpers.save_model_to_file(trainer, params["pytorch_output_path"])
def get_sarsa_parameters(self):
return ContinuousActionModelParameters(
rl=RLParameters(
gamma=DISCOUNT,
target_update_rate=1.0,
reward_burnin=100,
maxq_learning=False,
),
training=TrainingParameters(
layers=[-1, 256, 128, -1],
activations=["relu", "relu", "linear"],
minibatch_size=self.minibatch_size,
learning_rate=0.1,
optimizer="ADAM",
),
knn=KnnParameters(model_type="DQN"),
)
def run_gym(
params,
offline_train,
score_bar,
gpu_id,
save_timesteps_to_dataset=None,
start_saving_from_episode=0,
):
logger.info("Running gym with params")
logger.info(params)
rl_parameters = RLParameters(**params["rl"])
env_type = params["env"]
if offline_train:
# take random actions during data collection
epsilon = 1.0
else:
epsilon = rl_parameters.epsilon
env = OpenAIGymEnvironment(
env_type, epsilon, rl_parameters.softmax_policy, rl_parameters.gamma
)
replay_buffer = OpenAIGymMemoryPool(params["max_replay_memory_size"])
model_type = params["model_type"]
use_gpu = gpu_id != USE_CPU
trainer = create_trainer(params["model_type"], params, rl_parameters, use_gpu, env)
predictor = create_predictor(trainer, model_type, use_gpu)
c2_device = core.DeviceOption(
caffe2_pb2.CUDA if use_gpu else caffe2_pb2.CPU, int(gpu_id)
)
return train_sgd(
c2_device,
env,
offline_train,
replay_buffer,
model_type,
trainer,
predictor,
"{} test run".format(env_type),
score_bar,
**params["run_details"],
save_timesteps_to_dataset=save_timesteps_to_dataset,
start_saving_from_episode=start_saving_from_episode,
)
def test_sarsa_layer_validation(self):
env = Gridworld()
invalid_sarsa_params = DiscreteActionModelParameters(
actions=env.ACTIONS,
rl=RLParameters(gamma=DISCOUNT,
target_update_rate=0.5,
reward_burnin=10,
maxq_learning=False),
training=TrainingParameters(
layers=[-1, 3],
activations=['linear'],
minibatch_size=32,
learning_rate=0.1,
optimizer='SGD',
))
with self.assertRaises(Exception):
# layers[-1] should be 1
DiscreteActionTrainer(env.normalization, invalid_sarsa_params)
def main(args):
parser = argparse.ArgumentParser(
description="Train a RL net to play in an OpenAI Gym environment.")
parser.add_argument("-p",
"--parameters",
help="Path to JSON parameters file.")
parser.add_argument("-s",
"--score-bar",
help="Bar for averaged tests scores.",
type=float,
default=None)
parser.add_argument(
"-g",
"--gpu_id",
help="If set, will use GPU with specified ID. Otherwise will use CPU.",
default=USE_CPU)
args = parser.parse_args(args)
with open(args.parameters, 'r') as f:
params = json.load(f)
rl_settings = params['rl']
training_settings = params['training']
rl_settings['gamma'] = rl_settings['reward_discount_factor']
del rl_settings['reward_discount_factor']
training_settings['gamma'] = training_settings['learning_rate_decay']
del training_settings['learning_rate_decay']
env_type = params['env']
env = OpenAIGymEnvironment(env_type, rl_settings['epsilon'])
trainer_params = DiscreteActionModelParameters(
actions=env.actions,
rl=RLParameters(**rl_settings),
training=TrainingParameters(**training_settings))
device = core.DeviceOption(
caffe2_pb2.CPU if args.gpu_id == USE_CPU else caffe2_pb2.CUDA,
args.gpu_id)
with core.DeviceScope(device):
trainer = DiscreteActionTrainer(env.normalization,
trainer_params,
skip_normalization=True)
return run(env, trainer, "{} test run".format(env_type),
args.score_bar, **params["run_details"])
def test_trainer_maxq(self):
environment = GridworldContinuous()
rl_parameters = self.get_sarsa_parameters()
new_rl_parameters = ContinuousActionModelParameters(
rl=RLParameters(
gamma=DISCOUNT,
target_update_rate=0.5,
reward_burnin=10,
maxq_learning=True,
),
training=rl_parameters.training,
knn=rl_parameters.knn
)
maxq_trainer = ContinuousActionDQNTrainer(
new_rl_parameters,
environment.normalization,
environment.normalization_action,
)
states, actions, rewards, next_states, next_actions, is_terminal,\
possible_next_actions, reward_timelines = \
environment.generate_samples(100000, 1.0)
predictor = maxq_trainer.predictor()
tdps = environment.preprocess_samples(
states,
actions,
rewards,
next_states,
next_actions,
is_terminal,
possible_next_actions,
reward_timelines,
self.minibatch_size,
)
evaluator = GridworldContinuousEvaluator(environment, True)
self.assertGreater(evaluator.evaluate(predictor), 0.4)
for _ in range(2):
for tdp in tdps:
maxq_trainer.train_numpy(tdp, None)
evaluator.evaluate(predictor)
self.assertLess(evaluator.evaluate(predictor), 0.1)
def get_sac_parameters(self, use_2_q_functions=False):
return SACModelParameters(
rl=RLParameters(gamma=DISCOUNT,
target_update_rate=0.5,
reward_burnin=100),
training=SACTrainingParameters(
minibatch_size=self.minibatch_size,
use_2_q_functions=use_2_q_functions,
q_network_optimizer=OptimizerParameters(),
value_network_optimizer=OptimizerParameters(),
actor_network_optimizer=OptimizerParameters(),
),
q_network=FeedForwardParameters(layers=[128, 64],
activations=["relu", "relu"]),
value_network=FeedForwardParameters(layers=[128, 64],
activations=["relu", "relu"]),
actor_network=FeedForwardParameters(layers=[128, 64],
activations=["relu", "relu"]),
)
def get_sarsa_parameters(self):
return ContinuousActionModelParameters(
rl=RLParameters(
gamma=DISCOUNT,
target_update_rate=1.0,
reward_burnin=100,
maxq_learning=False,
),
training=TrainingParameters(
layers=[-1, 256, 128, -1],
activations=["relu", "relu", "linear"],
minibatch_size=self.minibatch_size,
learning_rate=0.05,
optimizer="ADAM",
),
knn=KnnParameters(model_type="DQN"),
rainbow=RainbowDQNParameters(double_q_learning=True,
dueling_architecture=False),
in_training_cpe=InTrainingCPEParameters(mdp_sampled_rate=0.1),
)
def get_sarsa_trainer_reward_boost(self, environment, reward_shape):
rl_parameters = RLParameters(
gamma=DISCOUNT,
target_update_rate=0.5,
reward_burnin=10,
maxq_learning=False,
reward_boost=reward_shape,
)
training_parameters = TrainingParameters(
layers=[-1, -1],
activations=["linear"],
minibatch_size=self.minibatch_size,
learning_rate=0.01,
optimizer="ADAM",
)
return DiscreteActionTrainer(
DiscreteActionModelParameters(
actions=environment.ACTIONS,
rl=rl_parameters,
training=training_parameters,
),
environment.normalization,
)
def get_sarsa_parameters(self, environment, reward_shape, dueling, clip_grad_norm):
rl_parameters = RLParameters(
gamma=DISCOUNT,
target_update_rate=1.0,
maxq_learning=False,
reward_boost=reward_shape,
)
training_parameters = TrainingParameters(
layers=[-1, 128, -1] if dueling else [-1, -1],
activations=["relu", "linear"] if dueling else ["linear"],
minibatch_size=self.minibatch_size,
learning_rate=0.05,
optimizer="ADAM",
clip_grad_norm=clip_grad_norm,
)
return DiscreteActionModelParameters(
actions=environment.ACTIONS,
rl=rl_parameters,
training=training_parameters,
rainbow=RainbowDQNParameters(
double_q_learning=True, dueling_architecture=dueling
),
)
def test_trainer_maxq(self):
env = Env(self.state_dims, self.action_dims)
maxq_parameters = DiscreteActionModelParameters(
actions=env.actions,
rl=RLParameters(gamma=0.95,
target_update_rate=0.9,
maxq_learning=True),
rainbow=RainbowDQNParameters(double_q_learning=True,
dueling_architecture=False),
training=TrainingParameters(
layers=self.layers,
activations=self.activations,
minibatch_size=1024,
learning_rate=0.25,
optimizer="ADAM",
),
)
# Q value should converge to very close to 20
trainer = self._train(maxq_parameters, env)
avg_q_value_after_training = torch.mean(trainer.all_action_scores)
self.assertLess(avg_q_value_after_training, 22)
self.assertGreater(avg_q_value_after_training, 18)
def get_ddpg_parameters(self):
return DDPGModelParameters(
rl=RLParameters(gamma=DISCOUNT,
target_update_rate=0.5,
maxq_learning=True),
shared_training=DDPGTrainingParameters(
minibatch_size=self.minibatch_size,
final_layer_init=0.003,
optimizer="ADAM",
),
actor_training=DDPGNetworkParameters(
layers=[-1, 256, 128, -1],
activations=["relu", "relu", "tanh"],
learning_rate=0.05,
l2_decay=0.01,
),
critic_training=DDPGNetworkParameters(
layers=[-1, 256, 256, 128, -1],
activations=["relu", "relu", "relu", "linear"],
learning_rate=0.05,
l2_decay=0.01,
),
)
def run_gym(params, score_bar, gpu_id):
rl_settings = params['rl']
training_settings = params['training']
rl_settings['gamma'] = rl_settings['reward_discount_factor']
del rl_settings['reward_discount_factor']
training_settings['gamma'] = training_settings['learning_rate_decay']
del training_settings['learning_rate_decay']
env_type = params['env']
env = OpenAIGymEnvironment(env_type, rl_settings['epsilon'])
trainer_params = DiscreteActionModelParameters(
actions=env.actions,
rl=RLParameters(**rl_settings),
training=TrainingParameters(**training_settings))
device = core.DeviceOption(
caffe2_pb2.CPU if gpu_id == USE_CPU else caffe2_pb2.CUDA,
gpu_id,
)
with core.DeviceScope(device):
if env.img:
trainer = DiscreteActionConvTrainer(
DiscreteActionConvModelParameters(
fc_parameters=trainer_params,
cnn_parameters=CNNModelParameters(**params['cnn']),
num_input_channels=env.num_input_channels,
img_height=env.height,
img_width=env.width),
env.normalization,
)
else:
trainer = DiscreteActionTrainer(
trainer_params,
env.normalization,
)
return run(env, trainer, "{} test run".format(env_type), score_bar,
**params["run_details"])
def get_sac_parameters(
self,
use_2_q_functions=False,
logged_action_uniform_prior=True,
constrain_action_sum=False,
):
return SACModelParameters(
rl=RLParameters(gamma=DISCOUNT, target_update_rate=0.5),
training=SACTrainingParameters(
minibatch_size=self.minibatch_size,
use_2_q_functions=use_2_q_functions,
q_network_optimizer=OptimizerParameters(),
value_network_optimizer=OptimizerParameters(),
actor_network_optimizer=OptimizerParameters(),
logged_action_uniform_prior=logged_action_uniform_prior,
),
q_network=FeedForwardParameters(layers=[128, 64],
activations=["relu", "relu"]),
value_network=FeedForwardParameters(layers=[128, 64],
activations=["relu", "relu"]),
actor_network=FeedForwardParameters(layers=[128, 64],
activations=["relu", "relu"]),
constrain_action_sum=constrain_action_sum,
)
def get_sarsa_trainer_reward_boost(
self,
environment,
reward_shape,
dueling,
use_gpu=False,
use_all_avail_gpus=False,
):
rl_parameters = RLParameters(
gamma=DISCOUNT,
target_update_rate=1.0,
reward_burnin=10,
maxq_learning=False,
reward_boost=reward_shape,
)
training_parameters = TrainingParameters(
layers=[-1, 128, -1] if dueling else [-1, -1],
activations=["relu", "linear"] if dueling else ["linear"],
minibatch_size=self.minibatch_size,
learning_rate=0.05,
optimizer="ADAM",
)
return DQNTrainer(
DiscreteActionModelParameters(
actions=environment.ACTIONS,
rl=rl_parameters,
training=training_parameters,
rainbow=RainbowDQNParameters(
double_q_learning=True, dueling_architecture=dueling
),
in_training_cpe=InTrainingCPEParameters(mdp_sampled_rate=0.1),
),
environment.normalization,
use_gpu=use_gpu,
use_all_avail_gpus=use_all_avail_gpus,
)
def run_gym(params, score_bar, gpu_id, save_timesteps_to_dataset=None):
logger.info("Running gym with params")
logger.info(params)
rl_parameters = RLParameters(**params["rl"])
env_type = params["env"]
env = OpenAIGymEnvironment(
env_type,
rl_parameters.epsilon,
rl_parameters.softmax_policy,
params["max_replay_memory_size"],
)
model_type = params["model_type"]
c2_device = core.DeviceOption(
caffe2_pb2.CPU if gpu_id == USE_CPU else caffe2_pb2.CUDA, gpu_id
)
if model_type == ModelType.DISCRETE_ACTION.value:
with core.DeviceScope(c2_device):
training_settings = params["training"]
training_parameters = TrainingParameters(**training_settings)
if env.img:
assert (
training_parameters.cnn_parameters is not None
), "Missing CNN parameters for image input"
training_parameters.cnn_parameters = CNNParameters(
**training_settings["cnn_parameters"]
)
training_parameters.cnn_parameters.conv_dims[0] = env.num_input_channels
training_parameters.cnn_parameters.input_height = env.height
training_parameters.cnn_parameters.input_width = env.width
training_parameters.cnn_parameters.num_input_channels = (
env.num_input_channels
)
else:
assert (
training_parameters.cnn_parameters is None
), "Extra CNN parameters for non-image input"
trainer_params = DiscreteActionModelParameters(
actions=env.actions, rl=rl_parameters, training=training_parameters
)
trainer = DiscreteActionTrainer(trainer_params, env.normalization)
elif model_type == ModelType.PARAMETRIC_ACTION.value:
with core.DeviceScope(c2_device):
training_settings = params["training"]
training_parameters = TrainingParameters(**training_settings)
if env.img:
assert (
training_parameters.cnn_parameters is not None
), "Missing CNN parameters for image input"
training_parameters.cnn_parameters = CNNParameters(
**training_settings["cnn_parameters"]
)
training_parameters.cnn_parameters.conv_dims[0] = env.num_input_channels
else:
assert (
training_parameters.cnn_parameters is None
), "Extra CNN parameters for non-image input"
trainer_params = ContinuousActionModelParameters(
rl=rl_parameters,
training=training_parameters,
knn=KnnParameters(model_type="DQN"),
)
trainer = ContinuousActionDQNTrainer(
trainer_params, env.normalization, env.normalization_action
)
elif model_type == ModelType.CONTINUOUS_ACTION.value:
training_settings = params["shared_training"]
actor_settings = params["actor_training"]
critic_settings = params["critic_training"]
trainer_params = DDPGModelParameters(
rl=rl_parameters,
shared_training=DDPGTrainingParameters(**training_settings),
actor_training=DDPGNetworkParameters(**actor_settings),
critic_training=DDPGNetworkParameters(**critic_settings),
)
# DDPG can handle continuous and discrete action spaces
if env.action_type == EnvType.CONTINUOUS_ACTION:
action_range = env.action_space.high
else:
action_range = None
trainer = DDPGTrainer(
trainer_params,
env.normalization,
env.normalization_action,
use_gpu=False,
action_range=action_range,
)
else:
raise NotImplementedError("Model of type {} not supported".format(model_type))
return run(
c2_device,
env,
model_type,
trainer,
"{} test run".format(env_type),
score_bar,
**params["run_details"],
save_timesteps_to_dataset=save_timesteps_to_dataset,
)
def train_network(params):
writer = None
if params["model_output_path"] is not None:
writer = SummaryWriter(log_dir=params["model_output_path"])
logger.info("Running DQN workflow with params:")
logger.info(params)
# Set minibatch size based on # of devices being used to train
params["training"]["minibatch_size"] *= minibatch_size_multiplier(
params["use_gpu"], params["use_all_avail_gpus"])
action_names = np.array(params["actions"])
rl_parameters = RLParameters(**params["rl"])
training_parameters = TrainingParameters(**params["training"])
rainbow_parameters = RainbowDQNParameters(**params["rainbow"])
trainer_params = DiscreteActionModelParameters(
actions=params["actions"],
rl=rl_parameters,
training=training_parameters,
rainbow=rainbow_parameters,
)
dataset = JSONDataset(params["training_data_path"],
batch_size=training_parameters.minibatch_size)
eval_dataset = JSONDataset(params["eval_data_path"], batch_size=16)
state_normalization = read_norm_file(params["state_norm_data_path"])
num_batches = int(len(dataset) / training_parameters.minibatch_size)
logger.info("Read in batch data set {} of size {} examples. Data split "
"into {} batches of size {}.".format(
params["training_data_path"],
len(dataset),
num_batches,
training_parameters.minibatch_size,
))
trainer = DQNTrainer(
trainer_params,
state_normalization,
use_gpu=params["use_gpu"],
use_all_avail_gpus=params["use_all_avail_gpus"],
)
trainer = update_model_for_warm_start(trainer)
preprocessor = Preprocessor(state_normalization, False)
evaluator = Evaluator(
trainer_params.actions,
trainer_params.rl.gamma,
trainer,
metrics_to_score=trainer.metrics_to_score,
)
start_time = time.time()
for epoch in range(int(params["epochs"])):
dataset.reset_iterator()
for batch_idx in range(num_batches):
report_training_status(batch_idx, num_batches, epoch,
int(params["epochs"]))
batch = dataset.read_batch(batch_idx)
tdp = preprocess_batch_for_training(preprocessor, batch,
action_names)
tdp.set_type(trainer.dtype)
trainer.train(tdp)
eval_dataset.reset_iterator()
accumulated_edp = None
while True:
batch = eval_dataset.read_batch(batch_idx)
if batch is None:
break
tdp = preprocess_batch_for_training(preprocessor, batch,
action_names)
edp = EvaluationDataPage.create_from_tdp(tdp, trainer)
if accumulated_edp is None:
accumulated_edp = edp
else:
accumulated_edp = accumulated_edp.append(edp)
accumulated_edp = accumulated_edp.compute_values(trainer.gamma)
cpe_start_time = time.time()
details = evaluator.evaluate_post_training(accumulated_edp)
details.log()
logger.info("CPE evaluation took {} seconds.".format(time.time() -
cpe_start_time))
through_put = (len(dataset) * int(params["epochs"])) / (time.time() -
start_time)
logger.info("Training finished. Processed ~{} examples / s.".format(
round(through_put)))
if writer is not None:
writer.close()
return export_trainer_and_predictor(trainer, params["model_output_path"])
