def get_optimizer_fn(hparams=None):
"""Returns a function `optimizer_fn` of making optimizer instance, along
with the optimizer class.
.. role:: python(code)
:language: python
The function has the signiture
:python:`optimizer_fn(learning_rate=None) -> optimizer class instance`
See the :attr:`"optimizer"` field of
:meth:`~texar.core.default_optimization_hparams` for all
hyperparameters and default values.
The optimizer class must be a subclass of
:tf_main:`tf.train.Optimizer <train/Optimizer>`.
Args:
hparams (dict or HParams, optional): hyperparameters. Missing
hyperparameters are set to default values automatically.
Returns:
- If hparams["type"] is a string or optimizer class, returns\
`(optimizer_fn, optimizer class)`,
- If hparams["type"] is an optimizer instance, returns \
`(the optimizer instance, optimizer class)`
"""
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams, default_optimization_hparams()["optimizer"])
opt = hparams["type"]
if isinstance(opt, tf.train.Optimizer):
return opt, type(opt)
opt_modules = [
'tensorflow.train', 'tensorflow.contrib.opt',
'texar.core.optimization', 'texar.custom'
]
try:
opt_class = utils.check_or_get_class(opt, opt_modules,
tf.train.Optimizer)
except TypeError:
raise ValueError(
"Unrecognized optimizer. Must be string name of the "
"optimizer class, or the class which is a subclass of "
"tf.train.Optimizer, or an instance of the subclass of "
"Optimizer.")
def _get_opt(learning_rate=None):
opt_kwargs = hparams["kwargs"].todict()
fn_args = set(utils.get_args(opt_class.__init__))
if 'learning_rate' in fn_args and learning_rate is not None:
opt_kwargs["learning_rate"] = learning_rate
return opt_class(**opt_kwargs)
return _get_opt, opt_class
def get_optimizer_fn(hparams=None):
"""Returns a function of making optimizer instance, along with the
optimizer class.
The function has the signiture:
(learning_rate=None) -> instance of the optimizer class,
The optimizer class must be a subclass of :tf_main:`~tf.train.Optimizer`.
See the :attr:`"optimizer"` field in
:meth:`~texar.core.optimization.default_optimization_hparams` for all
hyperparameters and default values.
If :attr:`hparams["type"]` is an optimier instance, returns the instance
directly.
Args:
hparams (dict or HParams, optional): hyperparameters. Missing
hyperparameters are set to default values automatically.
Returns:
(function that creates optimizer instance, optimizer class),
or the optimizer instance.
"""
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams, default_optimization_hparams()["optimizer"])
opt = hparams["type"]
if isinstance(opt, tf.train.Optimizer):
return opt, type(opt)
else:
opt_modules = [
'tensorflow.train', 'tensorflow.contrib.opt', 'texar.custom'
]
try:
opt_class = utils.check_or_get_class(opt, opt_modules,
tf.train.Optimizer)
except TypeError:
raise ValueError(
"Unrecognized optimizer. Must be string name of the "
"optimizer class, or the class which is a subclass of "
"tf.train.Optimizer, or an instance of the subclass of "
"Optimizer.")
def _get_opt(learning_rate=None):
opt_kwargs = hparams["kwargs"].todict()
fn_args = set(inspect.getargspec(opt_class.__init__).args)
if 'learning_rate' in fn_args and learning_rate is not None:
opt_kwargs["learning_rate"] = learning_rate
return opt_class(**opt_kwargs)
return _get_opt, opt_class
def get_scheduler(optimizer: Optimizer,
hparams: Optional[Union[HParams, Dict[str, Any]]] = None) -> \
Optional[_LRScheduler]:
r"""Creates a scheduler instance.
Args:
optimizer: A :torch_docs:`torch.optim.Optimizer
<optim.html#torch.optim.Optimizer>` instance.
hparams (dict or HParams, optional): hyperparameters. Missing
hyperparameters are set to default values automatically. See
:func:`~texar.core.default_optimization_hparams` for
all hyperparameters and default values.
:return:
A :torch_docs:`torch.optim.lr_scheduler._LRScheduler
<optim.html#how-to-adjust-learning-rate>` instance.
"""
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams, default_optimization_hparams())
hparams_scheduler = hparams["learning_rate_decay"]
scheduler_type = hparams_scheduler["type"]
if scheduler_type == "" or scheduler_type is None:
scheduler = None
else:
if isinstance(scheduler_type, _LRScheduler):
scheduler_class = scheduler_type
else:
scheduler_modules = ['torch.optim.lr_scheduler', 'texar.custom']
try:
scheduler_class = utils.check_or_get_class( # type: ignore
scheduler_type, scheduler_modules, _LRScheduler)
except TypeError:
raise ValueError(
"Unrecognized lr_scheduler. Must be string name of the "
"lr_scheduler class, or the class which is a subclass of "
"torch.optim._LRScheduler.")
scheduler_kwargs = hparams_scheduler["kwargs"].todict()
scheduler_kwargs.update({"optimizer": optimizer})
scheduler = scheduler_class(**scheduler_kwargs) # type: ignore
return scheduler
def get_optimizer(
params: Union[List[torch.Tensor], List[Dict[str, List[torch.Tensor]]]],
hparams: Optional[Union[HParams, Dict[str, Any]]] = None) -> \
Optimizer:
r"""Creates a optimizer instance.
Args:
params: an iterable of :class:`torch.Tensor` or
:class:`dict`. Specifies what Tensors should be optimized.
hparams (dict or HParams, optional): hyperparameters. Missing
hyperparameters are set to default values automatically. See
:func:`~texar.core.default_optimization_hparams` for
all hyperparameters and default values.
:return:
The :torch_docs:`torch.optim.Optimizer
<optim.html#torch.optim.Optimizer>` instance specified in
:attr:`hparams`.
"""
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams, default_optimization_hparams())
hparams_opt = hparams["optimizer"]
optimizer_type = hparams_opt["type"]
if isinstance(optimizer_type, Optimizer):
optimizer_class = optimizer_type
else:
optimizer_modules = ['torch.optim', 'texar.custom']
try:
optimizer_class = utils.check_or_get_class( # type: ignore
optimizer_type, optimizer_modules, Optimizer)
except TypeError:
raise ValueError(
"Unrecognized optimizer. Must be string name of the "
"optimizer class, or the class which is a subclass of "
"torch.optim.Optimizer, or an instance of the subclass of "
"Optimizer.")
optimizer_kwargs = hparams_opt["kwargs"].todict()
optimizer_kwargs.update({"params": params})
optimizer = optimizer_class(**optimizer_kwargs) # type: ignore
return optimizer
def get_layer(hparams: Union[HParams, Dict[str, Any]]) -> nn.Module:
r"""Makes a layer instance.
The layer must be an instance of :torch_nn:`Module`.
Args:
hparams (dict or HParams): Hyperparameters of the layer, with
structure:
.. code-block:: python
{
"type": "LayerClass",
"kwargs": {
# Keyword arguments of the layer class
# ...
}
}
Here:
`"type"`: str or layer class or layer instance
The layer type. This can be
- The string name or full module path of a layer class. If
the class name is provided, the class must be in module
:torch_nn:`Module`, :mod:`texar.core`, or :mod:`texar.custom`.
- A layer class.
- An instance of a layer class.
For example
.. code-block:: python
"type": "Conv1D" ..no # class name
"type": "texar.core.MaxReducePooling1D" # module path
"type": "my_module.MyLayer" # module path
"type": torch.nn.Module.Linear # class
"type": Conv1D(filters=10, kernel_size=2) # cell instance
"type": MyLayer(...) # cell instance
`"kwargs"`: dict
A dictionary of keyword arguments for constructor of the
layer class. Ignored if :attr:`"type"` is a layer instance.
- Arguments named "activation" can be a callable, or a `str` of
the name or module path to the activation function.
- Arguments named "\*_regularizer" and "\*_initializer" can be a
class instance, or a `dict` of hyperparameters of respective
regularizers and initializers. See
- Arguments named "\*_constraint" can be a callable, or a `str`
of the name or full path to the constraint function.
Returns:
A layer instance. If ``hparams["type"]`` is a layer instance, returns it
directly.
Raises:
ValueError: If :attr:`hparams` is `None`.
ValueError: If the resulting layer is not an instance of
:torch_nn:`Module`.
"""
if hparams is None:
raise ValueError("`hparams` must not be `None`.")
layer_type = hparams["type"]
if not is_str(layer_type) and not isinstance(layer_type, type):
layer = layer_type
else:
layer_modules = ["torch.nn", "texar.core", "texar.custom"]
layer_class = utils.check_or_get_class(layer_type, layer_modules)
if isinstance(hparams, dict):
if (layer_class.__name__ == "Linear"
and "in_features" not in hparams["kwargs"]):
raise ValueError("\"in_features\" should be specified for "
"\"torch.nn.{}\"".format(
layer_class.__name__))
elif (layer_class.__name__ in ["Conv1d", "Conv2d", "Conv3d"]
and "in_channels" not in hparams["kwargs"]):
raise ValueError("\"in_channels\" should be specified for "
"\"torch.nn.{}\"".format(
layer_class.__name__))
default_kwargs = _layer_class_to_default_kwargs_map.get(
layer_class, {})
default_hparams = {"type": layer_type, "kwargs": default_kwargs}
hparams = HParams(hparams, default_hparams)
# this case needs to be handled separately because
# :torch_nn:`Sequential`
# does not accept kwargs
if layer_type == "Sequential":
names: List[str] = []
layer = nn.Sequential()
sub_hparams = hparams.kwargs.layers
for hparam in sub_hparams:
sub_layer = get_layer(hparam)
name = utils.uniquify_str(sub_layer._get_name(), names)
names.append(name)
layer.add_module(name=name, module=sub_layer)
else:
layer = utils.get_instance(layer_type, hparams.kwargs.todict(),
layer_modules)
if not isinstance(layer, nn.Module):
raise ValueError("layer must be an instance of `torch.nn.Module`.")
return layer
def get_layer(hparams):
"""Makes a layer instance.
The layer must be an instance of :tf_main:`tf.layers.Layer <layers/Layer>`.
Args:
hparams (dict or HParams): Hyperparameters of the layer, with
structure:
.. code-block:: python
{
"type": "LayerClass",
"kwargs": {
# Keyword arguments of the layer class
# ...
}
}
Here:
"type" : str or layer class or layer instance
The layer type. This can be
- The string name or full module path of a layer class. If \
the class name is provided, the class must be in module \
:tf_main:`tf.layers <layers>`, :mod:`texar.core`, \
or :mod:`texar.custom`.
- A layer class.
- An instance of a layer class.
For example
.. code-block:: python
"type": "Conv1D" # class name
"type": "texar.core.MaxReducePooling1D" # module path
"type": "my_module.MyLayer" # module path
"type": tf.layers.Conv2D # class
"type": Conv1D(filters=10, kernel_size=2) # cell instance
"type": MyLayer(...) # cell instance
"kwargs" : dict
A dictionary of keyword arguments for constructor of the
layer class. Ignored if :attr:`"type"` is a layer instance.
- Arguments named "activation" can be a callable, \
or a `str` of \
the name or module path to the activation function.
- Arguments named "*_regularizer" and "*_initializer" \
can be a class instance, or a `dict` of \
hyperparameters of \
respective regularizers and initializers. See
- Arguments named "*_constraint" can be a callable, or a `str` \
of the name or full path to the constraint function.
Returns:
A layer instance. If hparams["type"] is a layer instance, returns it
directly.
Raises:
ValueError: If :attr:`hparams` is `None`.
ValueError: If the resulting layer is not an instance of
:tf_main:`tf.layers.Layer <layers/Layer>`.
"""
if hparams is None:
raise ValueError("`hparams` must not be `None`.")
layer_type = hparams["type"]
if not is_str(layer_type) and not isinstance(layer_type, type):
layer = layer_type
else:
layer_modules = ["tensorflow.layers", "texar.core", "texar.costum"]
layer_class = utils.check_or_get_class(layer_type, layer_modules)
if isinstance(hparams, dict):
default_kwargs = _layer_class_to_default_kwargs_map.get(
layer_class, {})
default_hparams = {"type": layer_type, "kwargs": default_kwargs}
hparams = HParams(hparams, default_hparams)
kwargs = {}
for k, v in hparams.kwargs.items():
if k.endswith('_regularizer'):
kwargs[k] = get_regularizer(v)
elif k.endswith('_initializer'):
kwargs[k] = get_initializer(v)
elif k.endswith('activation'):
kwargs[k] = get_activation_fn(v)
elif k.endswith('_constraint'):
kwargs[k] = get_constraint_fn(v)
else:
kwargs[k] = v
layer = utils.get_instance(layer_type, kwargs, layer_modules)
if not isinstance(layer, tf.layers.Layer):
raise ValueError("layer must be an instance of `tf.layers.Layer`.")
return layer
