def _initialize_cpe( self, reward_network, q_network_cpe, q_network_cpe_target, optimizer: Optimizer__Union, ) -> None: if self.calc_cpe_in_training: assert reward_network is not None, "reward_network is required for CPE" # pyre-fixme[16]: `RLTrainer` has no attribute `reward_network`. self.reward_network = reward_network # pyre-fixme[16]: `RLTrainer` has no attribute `reward_network_optimizer`. self.reward_network_optimizer = optimizer.make_optimizer( self.reward_network.parameters()) assert ( q_network_cpe is not None and q_network_cpe_target is not None ), "q_network_cpe and q_network_cpe_target are required for CPE" # pyre-fixme[16]: `RLTrainer` has no attribute `q_network_cpe`. self.q_network_cpe = q_network_cpe # pyre-fixme[16]: `RLTrainer` has no attribute `q_network_cpe_target`. self.q_network_cpe_target = q_network_cpe_target # pyre-fixme[16]: `RLTrainer` has no attribute `q_network_cpe_optimizer`. self.q_network_cpe_optimizer = optimizer.make_optimizer( self.q_network_cpe.parameters()) num_output_nodes = len(self.metrics_to_score) * self.num_actions # pyre-fixme[16]: `RLTrainer` has no attribute `reward_idx_offsets`. self.reward_idx_offsets = torch.arange( 0, num_output_nodes, self.num_actions, device=self.device, dtype=torch.long, ) else: self.reward_network = None
def __init__( self, seq2slate_net: Seq2SlateTransformerNet, minibatch_size: int = 1024, parameters: Seq2SlateParameters = field( # noqa: B008 default_factory=Seq2SlateParameters), baseline_net: Optional[BaselineNet] = None, baseline_warmup_num_batches: int = 0, use_gpu: bool = False, policy_optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), baseline_optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), ) -> None: self.seq2slate_net = seq2slate_net self.parameters = parameters self.use_gpu = use_gpu self.minibatch_size = minibatch_size self.minibatch = 0 self.baseline_net = baseline_net self.baseline_warmup_num_batches = baseline_warmup_num_batches self.rl_opt = policy_optimizer.make_optimizer( self.seq2slate_net.parameters()) if self.baseline_net: self.baseline_opt = baseline_optimizer.make_optimizer( # pyre-fixme[16]: `Optional` has no attribute `parameters`. self.baseline_net.parameters()) assert (self.parameters.importance_sampling_clamp_max is None or not self.parameters.on_policy), ( "importance_sampling_clamp_max is not useful and should " "be set to None in on-policy learning")
def __init__( self, q_network, q_network_target, reward_network, use_gpu: bool = False, # Start ParametricDQNTrainerParameters rl: rlp.RLParameters = field( default_factory=rlp.RLParameters), # noqa: B008 double_q_learning: bool = True, minibatch_size: int = 1024, minibatches_per_step: int = 1, optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), ) -> None: super().__init__(rl, use_gpu=use_gpu) self.double_q_learning = double_q_learning self.minibatch_size = minibatch_size self.minibatches_per_step = minibatches_per_step or 1 self.q_network = q_network self.q_network_target = q_network_target self.q_network_optimizer = optimizer.make_optimizer( self.q_network.parameters()) self.reward_network = reward_network self.reward_network_optimizer = optimizer.make_optimizer( self.reward_network.parameters())
def __init__( self, seq2slate_net: Seq2SlateTransformerNet, minibatch_size: int = 1024, parameters: Seq2SlateParameters = field( # noqa: B008 default_factory=Seq2SlateParameters), baseline_net: Optional[BaselineNet] = None, baseline_warmup_num_batches: int = 0, use_gpu: bool = False, policy_optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), baseline_optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), print_interval: int = 100, ) -> None: self.seq2slate_net = seq2slate_net self.parameters = parameters self.use_gpu = use_gpu self.print_interval = print_interval self.minibatch_size = minibatch_size self.minibatch = 0 self.baseline_net = baseline_net self.baseline_warmup_num_batches = baseline_warmup_num_batches self.rl_opt = policy_optimizer.make_optimizer( self.seq2slate_net.parameters()) if self.baseline_net: self.baseline_opt = baseline_optimizer.make_optimizer( # pyre-fixme[16]: `Optional` has no attribute `parameters`. self.baseline_net.parameters())
def __init__( self, policy: Policy, gamma: float = 0.0, optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default ), optimizer_value_net: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default ), off_policy: bool = False, reward_clip: float = 1e6, normalize: bool = True, subtract_mean: bool = True, offset_clamp_min: bool = False, update_freq: int = 100, # how many env steps between updates update_epochs: int = 5, # how many epochs to run when updating (for PPO) ppo_batch_size: int = 10, # batch size (number of trajectories) used for PPO updates ppo_epsilon: float = 0.2, # clamp importance weights between 1-epsilon and 1+epsilon entropy_weight: float = 0.0, # weight of the entropy term in the PPO loss value_net: Optional[ModelBase] = None, ): self.scorer = policy.scorer self.sampler = policy.sampler self.gamma = gamma self.optimizer_value_net = optimizer_value_net self.off_policy = off_policy self.reward_clip = reward_clip self.normalize = normalize self.subtract_mean = subtract_mean self.offset_clamp_min = offset_clamp_min self.update_freq = update_freq self.update_epochs = update_epochs self.ppo_batch_size = ppo_batch_size self.ppo_epsilon = ppo_epsilon self.entropy_weight = entropy_weight self.optimizer = optimizer.make_optimizer(self.scorer.parameters()) if value_net is not None: self.value_net = value_net self.value_net_optimizer = optimizer_value_net.make_optimizer( self.value_net.parameters() ) self.value_loss_fn = torch.nn.MSELoss(reduction="mean") else: self.value_net = None self.value_net_optimizer = None assert (ppo_epsilon >= 0) and ( ppo_epsilon <= 1 ), "ppo_epslion has to be in [0;1]" self.step = 0 self.traj_buffer = []
def __init__( self, q_network, q_network_target, metrics_to_score=None, reward_network=None, q_network_cpe=None, q_network_cpe_target=None, loss_reporter=None, use_gpu: bool = False, actions: List[str] = field(default_factory=list), # noqa: B008 rl: RLParameters = field(default_factory=RLParameters), # noqa: B008 double_q_learning: bool = True, num_atoms: int = 51, minibatch_size: int = 1024, minibatches_per_step: int = 1, optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), cpe_optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), evaluation: EvaluationParameters = field( # noqa: B008 default_factory=EvaluationParameters), ) -> None: super().__init__( rl, use_gpu=use_gpu, metrics_to_score=metrics_to_score, actions=actions, evaluation_parameters=evaluation, loss_reporter=loss_reporter, ) self.double_q_learning = double_q_learning self.minibatch_size = minibatch_size self.minibatches_per_step = minibatches_per_step self._actions = actions self.q_network = q_network self.q_network_target = q_network_target self.q_network_optimizer = optimizer.make_optimizer( self.q_network.parameters()) self.num_atoms = num_atoms self.quantiles = ( (0.5 + torch.arange(self.num_atoms, device=self.device).float()) / float(self.num_atoms)).view(1, -1) self._initialize_cpe(reward_network, q_network_cpe, q_network_cpe_target, optimizer=cpe_optimizer) self.reward_boosts = torch.zeros([1, len(self._actions)], device=self.device) if rl.reward_boost is not None: # pyre-fixme[16]: Optional type has no attribute `keys`. for k in rl.reward_boost.keys(): i = self._actions.index(k) # pyre-fixme[16]: Optional type has no attribute `__getitem__`. self.reward_boosts[0, i] = rl.reward_boost[k]
def __init__( self, q_network, q_network_target, use_gpu: bool = False, # Start SlateQTrainerParameters rl: rlp.RLParameters = field( # noqa: B008 default_factory=lambda: rlp.RLParameters(maxq_learning=False)), optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), single_selection: bool = True, minibatch_size: int = 1024, evaluation: rlp.EvaluationParameters = field( # noqa: B008 default_factory=lambda: rlp.EvaluationParameters( calc_cpe_in_training=False)), ) -> None: super().__init__(rl, use_gpu=use_gpu) self.minibatches_per_step = 1 self.minibatch_size = minibatch_size self.single_selection = single_selection self.q_network = q_network self.q_network_target = q_network_target self.q_network_optimizer = optimizer.make_optimizer( self.q_network.parameters())
def __init__( self, actor_network, q1_network, q2_network=None, use_gpu: bool = False, # Start TD3TrainerParameters rl: RLParameters = field(default_factory=RLParameters), # noqa: B008 q_network_optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), actor_network_optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), minibatch_size: int = 64, use_2_q_functions: bool = True, noise_variance: float = 0.2, noise_clip: float = 0.5, delayed_policy_update: int = 2, minibatches_per_step: int = 1, ) -> None: """ Args: TODO: fill in """ super().__init__(rl, use_gpu=use_gpu) self.minibatch_size = minibatch_size self.minibatches_per_step = minibatches_per_step or 1 self.q1_network = q1_network self.q1_network_target = copy.deepcopy(self.q1_network) self.q1_network_optimizer = q_network_optimizer.make_optimizer( q1_network.parameters()) self.q2_network = q2_network if self.q2_network is not None: self.q2_network_target = copy.deepcopy(self.q2_network) self.q2_network_optimizer = q_network_optimizer.make_optimizer( q2_network.parameters()) self.actor_network = actor_network self.actor_network_target = copy.deepcopy(self.actor_network) self.actor_network_optimizer = actor_network_optimizer.make_optimizer( actor_network.parameters()) self.noise_variance = noise_variance self.noise_clip_range = (-noise_clip, noise_clip) self.delayed_policy_update = delayed_policy_update
def __init__( self, q_network, q_network_target, metrics_to_score=None, loss_reporter=None, use_gpu: bool = False, actions: List[str] = field(default_factory=list), # noqa: B008 rl: RLParameters = field(default_factory=RLParameters), # noqa: B008 double_q_learning: bool = True, minibatch_size: int = 1024, minibatches_per_step: int = 1, num_atoms: int = 51, qmin: float = -100, qmax: float = 200, optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), evaluation: EvaluationParameters = field( # noqa: B008 default_factory=EvaluationParameters), ) -> None: RLTrainer.__init__( self, rl, use_gpu=use_gpu, metrics_to_score=metrics_to_score, actions=actions, loss_reporter=loss_reporter, ) self.double_q_learning = double_q_learning self.minibatch_size = minibatch_size self.minibatches_per_step = minibatches_per_step self._actions = actions self.q_network = q_network self.q_network_target = q_network_target self.q_network_optimizer = optimizer.make_optimizer( q_network.parameters()) self.qmin = qmin self.qmax = qmax self.num_atoms = num_atoms self.support = torch.linspace(self.qmin, self.qmax, self.num_atoms, device=self.device) self.scale_support = (self.qmax - self.qmin) / (self.num_atoms - 1.0) self.reward_boosts = torch.zeros([1, len(self._actions)], device=self.device) if rl.reward_boost is not None: # pyre-fixme[16]: Optional type has no attribute `keys`. for k in rl.reward_boost.keys(): i = self._actions.index(k) # pyre-fixme[16]: Optional type has no attribute `__getitem__`. self.reward_boosts[0, i] = rl.reward_boost[k]
def __init__( self, reward_net: ModelBase, use_gpu: bool = False, minibatch_size: int = 1024, optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), ) -> None: self.reward_net = reward_net self.use_gpu = use_gpu self.minibatch_size = minibatch_size self.minibatch = 0 self.loss_fn = torch.nn.MSELoss(reduction="mean") self.opt = optimizer.make_optimizer(self.reward_net.parameters())
def __init__( self, imitator, use_gpu: bool = False, rl: RLParameters = field(default_factory=RLParameters), # noqa: B008 minibatch_size: int = 1024, minibatches_per_step: int = 1, optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), ) -> None: super().__init__(rl, use_gpu=use_gpu) self.minibatch_size = minibatch_size self.minibatches_per_step = minibatches_per_step or 1 self.imitator = imitator self.imitator_optimizer = optimizer.make_optimizer( imitator.parameters())
def __init__( self, reward_net: ModelBase, use_gpu: bool = False, minibatch_size: int = 1024, optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), loss_type: LossFunction = LossFunction.MSE, ) -> None: self.reward_net = reward_net self.use_gpu = use_gpu self.minibatch_size = minibatch_size self.minibatch = 0 self.opt = optimizer.make_optimizer(self.reward_net.parameters()) self.loss_type = loss_type self.loss_fn = _get_loss_function(loss_type)
def __init__( self, reward_net: ModelBase, use_gpu: bool = False, optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), loss_type: LossFunction = LossFunction.MSE, reward_ignore_threshold: Optional[float] = None, weighted_by_inverse_propensity: bool = False, ) -> None: self.reward_net = reward_net self.use_gpu = use_gpu self.minibatch = 0 self.opt = optimizer.make_optimizer(self.reward_net.parameters()) self.loss_type = loss_type self.reward_ignore_threshold = reward_ignore_threshold self.weighted_by_inverse_propensity = weighted_by_inverse_propensity self.loss_fn = _get_loss_function(loss_type, reward_ignore_threshold, weighted_by_inverse_propensity)
def __init__( self, seq2slate_net: Seq2SlateTransformerNet, parameters: Seq2SlateParameters, minibatch_size: int, use_gpu: bool = False, policy_optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default ), ) -> None: self.parameters = parameters self.use_gpu = use_gpu self.seq2slate_net = seq2slate_net self.minibatch_size = minibatch_size self.minibatch = 0 self.optimizer = policy_optimizer.make_optimizer( self.seq2slate_net.parameters() ) self.kl_div_loss = nn.KLDivLoss(reduction="batchmean")
def __init__( self, seq2slate_net: Seq2SlateTransformerNet, parameters: Seq2SlateParameters, minibatch_size: int, use_gpu: bool = False, policy_optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), print_interval: int = 100, ) -> None: self.parameters = parameters self.use_gpu = use_gpu self.print_interval = print_interval self.seq2slate_net = seq2slate_net self.minibatch_size = minibatch_size self.minibatch = 0 self.optimizer = policy_optimizer.make_optimizer( self.seq2slate_net.parameters()) # TODO: T62269969 add baseline_net in training self.kl_div_loss = nn.KLDivLoss(reduction="none")
def __init__( self, seq2slate_net: Seq2SlateTransformerNet, minibatch_size: int = 1024, loss_reporter=None, use_gpu: bool = False, policy_optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), ) -> None: self.loss_reporter = loss_reporter self.use_gpu = use_gpu self.seq2slate_net = seq2slate_net self.minibatch_size = minibatch_size self.minibatch = 0 self.optimizer = policy_optimizer.make_optimizer( self.seq2slate_net.parameters()) self.log_softmax = nn.LogSoftmax(dim=1) self.kl_loss = nn.KLDivLoss(reduction="batchmean") if self.loss_reporter is None: self.loss_reporter = NoOpLossReporter()
def __init__( self, q_network, q_network_target, reward_network, q_network_cpe=None, q_network_cpe_target=None, metrics_to_score=None, imitator=None, loss_reporter=None, use_gpu: bool = False, actions: List[str] = field(default_factory=list), # noqa: B008 rl: RLParameters = field(default_factory=RLParameters), # noqa: B008 double_q_learning: bool = True, bcq: Optional[BCQConfig] = None, minibatch_size: int = 1024, minibatches_per_step: int = 1, optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), evaluation: EvaluationParameters = field( # noqa: B008 default_factory=EvaluationParameters), ) -> None: super().__init__( rl, use_gpu=use_gpu, metrics_to_score=metrics_to_score, actions=actions, evaluation_parameters=evaluation, loss_reporter=loss_reporter, ) assert self._actions is not None, "Discrete-action DQN needs action names" self.double_q_learning = double_q_learning self.minibatch_size = minibatch_size self.minibatches_per_step = minibatches_per_step or 1 self.q_network = q_network self.q_network_target = q_network_target self.q_network_optimizer = optimizer.make_optimizer( q_network.parameters()) self._initialize_cpe(reward_network, q_network_cpe, q_network_cpe_target, optimizer=optimizer) # pyre-fixme[6]: Expected `Sized` for 1st param but got `Optional[List[str]]`. self.reward_boosts = torch.zeros([1, len(self._actions)], device=self.device) if rl.reward_boost is not None: # pyre-fixme[16]: `Optional` has no attribute `keys`. for k in rl.reward_boost.keys(): # pyre-fixme[16]: `Optional` has no attribute `index`. i = self._actions.index(k) # pyre-fixme[16]: `Optional` has no attribute `__getitem__`. self.reward_boosts[0, i] = rl.reward_boost[k] # Batch constrained q-learning self.bcq = bcq is not None if self.bcq: assert bcq is not None self.bcq_drop_threshold = bcq.drop_threshold self.bcq_imitator = imitator
def __init__( self, actor_network, q1_network, q2_network=None, value_network=None, use_gpu: bool = False, # Start SACTrainerParameters rl: RLParameters = field(default_factory=RLParameters), # noqa: B008 q_network_optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), value_network_optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), actor_network_optimizer: Optimizer__Union = field( # noqa: B008 default_factory=Optimizer__Union.default), alpha_optimizer: Optional[Optimizer__Union] = field( # noqa: B008 default_factory=Optimizer__Union.default), minibatch_size: int = 1024, entropy_temperature: float = 0.01, logged_action_uniform_prior: bool = True, target_entropy: float = -1.0, action_embedding_kld_weight: Optional[float] = None, apply_kld_on_mean: bool = False, action_embedding_mean: Optional[List[float]] = None, action_embedding_variance: Optional[List[float]] = None, ) -> None: """ Args: actor_network: states -> actions, trained to maximize soft value, which is value + policy entropy. q1_network: states, action -> q-value q2_network (optional): double q-learning to stabilize training from overestimation bias value_network (optional): states -> value of state under actor # alpha in the paper; controlling explore & exploit # TODO: finish """ super().__init__(rl, use_gpu=use_gpu) self.minibatch_size = minibatch_size self.minibatches_per_step = 1 self.q1_network = q1_network self.q1_network_optimizer = q_network_optimizer.make_optimizer( q1_network.parameters()) self.q2_network = q2_network if self.q2_network is not None: self.q2_network_optimizer = q_network_optimizer.make_optimizer( q2_network.parameters()) self.value_network = value_network if self.value_network is not None: self.value_network_optimizer = value_network_optimizer.make_optimizer( value_network.parameters()) self.value_network_target = copy.deepcopy(self.value_network) else: self.q1_network_target = copy.deepcopy(self.q1_network) self.q2_network_target = copy.deepcopy(self.q2_network) self.actor_network = actor_network self.actor_network_optimizer = actor_network_optimizer.make_optimizer( actor_network.parameters()) self.entropy_temperature = entropy_temperature self.alpha_optimizer = None device = "cuda" if use_gpu else "cpu" if alpha_optimizer is not None: self.target_entropy = target_entropy self.log_alpha = torch.tensor([np.log(self.entropy_temperature)], requires_grad=True, device=device) self.alpha_optimizer = alpha_optimizer.make_optimizer( [self.log_alpha]) self.logged_action_uniform_prior = logged_action_uniform_prior self.add_kld_to_loss = bool(action_embedding_kld_weight) self.apply_kld_on_mean = apply_kld_on_mean if self.add_kld_to_loss: self.kld_weight = action_embedding_kld_weight self.action_emb_mean = torch.tensor(action_embedding_mean, device=device) self.action_emb_variance = torch.tensor(action_embedding_variance, device=device)