def __init__(self, learning_rate: Union[float, Callable, Dict] = 1e-3, beta_1: Union[float, Callable] = 0.9, beta_2: Union[float, Callable] = 0.999, beta_3: Union[float, Callable] = 0.9, num_epochs: int = 1000, steps_per_epoch: int = 100, epsilon: float = 1e-8, use_softplus: bool = False, beta_softplus: float = 50.0, eps_clipping: float = 1e-3, threshold_clipping: float = 1e-2, weight_decay: float = 1e-4, beta_lookahead: float = 0.5, lookahead_every_nth_iter=5.0, nb_warmup_iterations=None, nb_warmdown_iterations=None, centralize_gradients: bool = True, normalize_gradients: bool = True, name='Ranger21', **kwargs): super(Ranger21, self).__init__(name, **kwargs) self.num_epochs = num_epochs self.steps_per_epoch = steps_per_epoch self.nb_iterations = float(num_epochs * steps_per_epoch) self.use_softplus = use_softplus self.centralize_gradients = centralize_gradients self.normalize_gradients = normalize_gradients if isinstance(learning_rate, Dict): learning_rate = deserialize(learning_rate) self._set_hyper('learning_rate', kwargs.get('lr', learning_rate)) self._set_hyper('beta_1', beta_1) self._set_hyper('beta_2', beta_2) self._set_hyper('beta_3', beta_3) self.epsilon = epsilon or backend_config.epsilon() self._set_hyper('nb_iterations', self.nb_iterations) self._set_hyper( 'nb_warmup_iterations', float(0.22 * self.nb_iterations) if nb_warmup_iterations is None else nb_warmup_iterations) self._set_hyper( 'nb_warmdown_iterations', float(0.28 * self.nb_iterations) if nb_warmdown_iterations is None else nb_warmdown_iterations) self._set_hyper('beta_softplus', beta_softplus) self._set_hyper('eps_clipping', eps_clipping) self._set_hyper('threshold_clipping', threshold_clipping) self._set_hyper('weight_decay', weight_decay) self._set_hyper('beta_lookahead', beta_lookahead) self._set_hyper('lookahead_every_nth_iter', lookahead_every_nth_iter)
def variables(self): """A list of variables which encode the current state of `Optimizer`. Includes slot variables and additional global variables created by the optimizer in the current default graph. Returns: A list of variables. """ if "lr" in config: config["learning_rate"] = config.pop("lr") if "learning_rate" in config: if isinstance(config["learning_rate"], dict): config["learning_rate"] = learning_rate_schedule.deserialize( config["learning_rate"]) return cls(**config)
def from_config(cls, config, custom_objects=None): """Creates an optimizer from its config. This method is the reverse of `get_config`, capable of instantiating the same optimizer from the config dictionary. Arguments: config: A Python dictionary, typically the output of get_config. custom_objects: A Python dictionary mapping names to additional Python objects used to create this optimizer, such as a function used for a hyperparameter. Returns: An optimizer instance. """ if "lr" in config: config["learning_rate"] = config.pop("lr") if "learning_rate" in config: if isinstance(config["learning_rate"], dict): config["learning_rate"] = learning_rate_schedule.deserialize( config["learning_rate"]) return cls(**config)
def _maybe_serialized(lr_decay, serialize_and_deserialize): if serialize_and_deserialize: serialized = learning_rate_schedule.serialize(lr_decay) return learning_rate_schedule.deserialize(serialized) else: return lr_decay
def _maybe_serialized(lr_decay, serialize_and_deserialize): if serialize_and_deserialize: serialized = learning_rate_schedule.serialize(lr_decay) return learning_rate_schedule.deserialize(serialized) else: return lr_decay