def create_parametric_dqn_trainer_from_params(
model: ContinuousActionModelParameters,
state_normalization_parameters: Dict[int, NormalizationParameters],
action_normalization_parameters: Dict[int, NormalizationParameters],
use_gpu: bool = False,
use_all_avail_gpus: bool = False,
):
q_network = FullyConnectedParametricDQN(
state_dim=get_num_output_features(state_normalization_parameters),
action_dim=get_num_output_features(action_normalization_parameters),
sizes=model.training.layers[1:-1],
activations=model.training.activations[:-1],
)
reward_network = FullyConnectedParametricDQN(
state_dim=get_num_output_features(state_normalization_parameters),
action_dim=get_num_output_features(action_normalization_parameters),
sizes=model.training.layers[1:-1],
activations=model.training.activations[:-1],
)
q_network_target = q_network.get_target_network()
if use_gpu and torch.cuda.is_available():
q_network = q_network.cuda()
q_network_target = q_network_target.cuda()
reward_network = reward_network.cuda()
if use_all_avail_gpus:
q_network = q_network.get_distributed_data_parallel_model()
q_network_target = q_network_target.get_distributed_data_parallel_model(
)
reward_network = reward_network.get_distributed_data_parallel_model()
return ParametricDQNTrainer(q_network, q_network_target, reward_network,
model, use_gpu)
def get_trainer(
self, environment, parameters=None, use_gpu=False, use_all_avail_gpus=False
):
parameters = parameters or self.get_sarsa_parameters()
q_network = FullyConnectedParametricDQN(
state_dim=get_num_output_features(environment.normalization),
action_dim=get_num_output_features(environment.normalization_action),
sizes=parameters.training.layers[1:-1],
activations=parameters.training.activations[:-1],
)
reward_network = FullyConnectedParametricDQN(
state_dim=get_num_output_features(environment.normalization),
action_dim=get_num_output_features(environment.normalization_action),
sizes=parameters.training.layers[1:-1],
activations=parameters.training.activations[:-1],
)
if use_gpu:
q_network = q_network.cuda()
reward_network = reward_network.cuda()
if use_all_avail_gpus:
q_network = q_network.get_distributed_data_parallel_model()
reward_network = reward_network.get_distributed_data_parallel_model()
q_network_target = q_network.get_target_network()
trainer = ParametricDQNTrainer(
q_network, q_network_target, reward_network, parameters
)
return trainer
def get_trainer(self,
environment,
parameters=None,
use_gpu=False,
use_all_avail_gpus=False):
layers = [256, 128]
activations = ["relu", "relu"]
parameters = parameters or self.get_sarsa_parameters()
q_network = FullyConnectedParametricDQN(
state_dim=get_num_output_features(environment.normalization),
action_dim=get_num_output_features(
environment.normalization_action),
sizes=layers,
activations=activations,
)
reward_network = FullyConnectedParametricDQN(
state_dim=get_num_output_features(environment.normalization),
action_dim=get_num_output_features(
environment.normalization_action),
sizes=layers,
activations=activations,
)
if use_gpu:
q_network = q_network.cuda()
reward_network = reward_network.cuda()
if use_all_avail_gpus:
q_network = q_network.get_distributed_data_parallel_model()
reward_network = reward_network.get_distributed_data_parallel_model(
)
q_network_target = q_network.get_target_network()
param_dict = parameters.asdict() # type: ignore
trainer = ParametricDQNTrainer(q_network, q_network_target,
reward_network, **param_dict)
return trainer
def create_parametric_dqn_trainer_from_params(
model: ContinuousActionModelParameters,
state_normalization_parameters: Dict[int, NormalizationParameters],
action_normalization_parameters: Dict[int, NormalizationParameters],
use_gpu: bool = False,
use_all_avail_gpus: bool = False,
):
q_network = FullyConnectedParametricDQN(
state_dim=get_num_output_features(state_normalization_parameters),
action_dim=get_num_output_features(action_normalization_parameters),
sizes=model.training.layers[1:-1],
activations=model.training.activations[:-1],
)
reward_network = FullyConnectedParametricDQN(
state_dim=get_num_output_features(state_normalization_parameters),
action_dim=get_num_output_features(action_normalization_parameters),
sizes=model.training.layers[1:-1],
activations=model.training.activations[:-1],
)
q_network_target = q_network.get_target_network()
if use_gpu:
q_network = q_network.cuda()
q_network_target = q_network_target.cuda()
reward_network = reward_network.cuda()
if use_all_avail_gpus:
q_network = q_network.get_distributed_data_parallel_model()
q_network_target = q_network_target.get_distributed_data_parallel_model(
)
reward_network = reward_network.get_distributed_data_parallel_model()
trainer_parameters = ParametricDQNTrainerParameters( # type: ignore
rl=model.rl,
double_q_learning=model.rainbow.double_q_learning,
minibatch_size=model.training.minibatch_size,
optimizer=OptimizerParameters(
optimizer=model.training.optimizer,
learning_rate=model.training.learning_rate,
l2_decay=model.training.l2_decay,
),
)
return ParametricDQNTrainer(
q_network,
q_network_target,
reward_network,
use_gpu=use_gpu,
**trainer_parameters.asdict() # type: ignore
)
def get_modular_sarsa_trainer_exporter(self,
environment,
parameters=None,
use_gpu=False,
use_all_avail_gpus=False):
parameters = parameters or self.get_sarsa_parameters()
q_network = FullyConnectedParametricDQN(
state_dim=get_num_output_features(environment.normalization),
action_dim=get_num_output_features(
environment.normalization_action),
sizes=parameters.training.layers[1:-1],
activations=parameters.training.activations[:-1],
)
reward_network = FullyConnectedParametricDQN(
state_dim=get_num_output_features(environment.normalization),
action_dim=get_num_output_features(
environment.normalization_action),
sizes=parameters.training.layers[1:-1],
activations=parameters.training.activations[:-1],
)
if use_gpu:
q_network = q_network.cuda()
reward_network = reward_network.cuda()
if use_all_avail_gpus:
q_network = q_network.get_distributed_data_parallel_model()
reward_network = reward_network.get_distributed_data_parallel_model(
)
q_network_target = q_network.get_target_network()
trainer = ParametricDQNTrainer(q_network, q_network_target,
reward_network, parameters)
feature_extractor = PredictorFeatureExtractor(
state_normalization_parameters=environment.normalization,
action_normalization_parameters=environment.normalization_action,
)
output_transformer = ParametricActionOutputTransformer()
exporter = ParametricDQNExporter(q_network, feature_extractor,
output_transformer)
return (trainer, exporter)