def _check_cudnn_hidden_init(s0, shape, nnops, name):
nb_layers, batch_size, hidden_size = shape
# ====== init s0 ====== #
if s0 is None and hasattr(nnops, name):
s0 = getattr(nnops, name)
elif s0 is not None:
if hasattr(s0, '__call__') or K.is_variable(s0) or \
isinstance(s0, np.ndarray):
_ = (nb_layers, 1, hidden_size) \
if hasattr(s0, '__call__') or isinstance(s0, np.ndarray) \
else s0.shape
s0 = nnops.config.create_params(s0,
shape=_,
name=name,
roles=InitialState)
# ====== check s0 shape ====== #
init_shape = s0.shape
if s0.shape.ndims == 2:
if s0.shape[-1].value != hidden_size:
raise ValueError(
'init state has %d dimension, but the hidden_size=%d' %
(init_shape[-1], hidden_size))
elif init_shape[::2] != (nb_layers, hidden_size):
raise ValueError('Require init states of size: %s, but '
'given state of size: %s' % (shape, init_shape))
# ====== return the right shape ====== #
setattr(nnops, name, s0)
return s0
def _check_cudnn_hidden_init(s0, shape, nnops, name):
nb_layers, batch_size, hidden_size = shape
# ====== init s0 ====== #
if s0 is None and hasattr(nnops, name):
s0 = getattr(nnops, name)
elif s0 is not None:
if hasattr(s0, '__call__') or K.is_variable(s0) or \
isinstance(s0, np.ndarray):
_ = (nb_layers, 1, hidden_size) \
if hasattr(s0, '__call__') or isinstance(s0, np.ndarray) \
else s0.shape
s0 = nnops.config.create_params(
s0, shape=_, name=name, roles=InitialState)
# ====== check s0 shape ====== #
init_shape = s0.shape
if s0.shape.ndims == 2:
if s0.shape[-1].value != hidden_size:
raise ValueError('init state has %d dimension, but the hidden_size=%d' %
(init_shape[-1], hidden_size))
elif init_shape[::2] != (nb_layers, hidden_size):
raise ValueError('Require init states of size: %s, but '
'given state of size: %s' % (shape, init_shape))
# ====== return the right shape ====== #
setattr(nnops, name, s0)
return s0
def _apply(self, x, **kwargs):
is_training = False
for i in as_tuple(x):
if K.is_variable(i) and K.is_training():
is_training = True
if is_training:
self.ops = [self.training]
return self.training(x, **_shrink_kwargs(self.training, kwargs))
else:
self.ops = [self.deploying]
return self.deploying(x, **_shrink_kwargs(self.deploying, kwargs))
def add_trainable_weights(layer, *variables):
from odin.backend import is_variable
variables = nest.flatten(variables)
assert all(is_variable(v) for v in variables), \
"All objects from variables must be instance of tensorflow.Variable"
assert isinstance(layer, Layer), \
"layer must be instance of tensorflow.python.keras.layers.Layer"
variables = [v for v in variables if v not in layer._trainable_weights]
layer._trainable_weights = layer._trainable_weights + variables
return layer
def _recurrsive_extract_shape(x):
shape_list = []
if not isinstance(x, (tuple, list)):
x = [x]
for i in x:
if K.is_variable(i):
shape = K.get_shape(i)
if isinstance(shape, (tuple, list)):
shape_list.append(shape)
elif isinstance(i, (tuple, list)):
shape_list += _recurrsive_extract_shape(i)
return shape_list
def add_trainable_weights(layer, *variables):
from odin.backend import is_variable
variables = nest.flatten(variables)
assert all(is_variable(v) for v in variables), \
"All objects from variables must be instance of tensorflow.Variable"
assert isinstance(layer, Layer), \
"layer must be instance of tensorflow.python.keras.layers.Layer"
variables = [v for v in variables
if v not in layer._trainable_weights]
layer._trainable_weights = layer._trainable_weights + variables
return layer
def _initialize_param(name, spec, shape):
""" return a ndarray or trainable_variable """
#####################################
# 0. initializing function.
if callable(spec):
spec = spec(shape)
#####################################
# 1. Shared variable, just check the shape.
if K.is_trainable_variable(spec):
spec_shape = K.get_shape(spec)
if not isinstance(spec_shape, tuple):
spec_shape = K.eval(spec_shape)
if shape is None:
shape = spec_shape
elif tuple(shape) != tuple(spec_shape):
raise Exception(
'Given variable has different shape from requirement'
', %s != %s' % (str(spec_shape), str(shape)))
#####################################
# 2. expression, we can only check number of dimension.
elif K.is_variable(spec):
# We cannot check the shape here, Theano expressions (even shared
# variables) do not have a fixed compile-time shape. We can check the
# dimensionality though.
# Note that we cannot assign a name here. We could assign to the
# `name` attribute of the variable, but the user may have already
# named the variable and we don't want to override this.
if shape is not None and K.ndim(spec) != len(shape):
raise Exception(
"parameter with name=%s has %d dimensions, should be "
"%d" % (name, spec.ndim, len(shape)))
#####################################
# 3. numpy ndarray, create shared variable wraper for it.
elif isinstance(spec, np.ndarray):
if shape is not None and spec.shape != shape:
raise RuntimeError(
"parameter with name=%s has shape %s, should be "
"%s" % (name, spec.shape, shape))
#####################################
# 5. Exception.
else:
raise RuntimeError("cannot initialize parameters: 'spec' is not "
"a numpy array, a Theano expression, or a "
"callable")
return spec, shape
def create_params(self, spec, shape, name, nnops, roles=[], nb_params=1):
"""
Parameters
----------
spec: variable, numpy.ndarray, function
specification for initializing the weights
shape: tuple, list
expected shape for given variable
name: str
name for the variable
nnops: NNOps
parent operator of this parameters
roles: odin.basic.VariableRole
categories of this variable
nb_params: int
number of parameters that horizontally stacked into
given `shape (e.g. nb_params=2, create 2 parameters with
given `shape and horizontally stack them into 1 parameters)
* do NOT support when `spec` is variable.
"""
if not isinstance(roles, (tuple, list)):
roles = [roles]
if not isinstance(nnops, NNOps):
raise Exception('nnops must be instance of odin.nnet.base.NNOps')
shape = tuple(shape) # convert to tuple if needed
if any(d <= 0 for d in shape):
raise ValueError(
("Cannot create param with a non-positive shape dimension. "
"Tried to create param with shape=%r, name=%r") %
(shape, name))
# ====== create parameters ====== #
spec = as_tuple(spec, nb_params)
spec = [_initialize_param(name, s, shape) for s in spec]
# check shape returned
shape = list(set([i[-1] for i in spec]))
if len(shape) > 1:
raise Exception(
'shape are inconsitent among all given "spec", the '
'created shape is: %s' % str(shape))
shape = shape[0]
# check spec returned
spec = [i[0] for i in spec]
if isinstance(spec[0], np.ndarray):
with K.variable_scope(nnops.name):
spec = np.concatenate(spec, axis=-1)
shape = spec.shape
spec = K.variable(spec, name=name)
elif K.is_trainable_variable(spec[0]):
if nb_params > 1:
with K.variable_scope(nnops.name):
spec = np.concatenate([K.get_value(i) for i in spec],
axis=-1)
shape = spec.shape
spec = K.variable(spec, name=name)
else:
spec = spec[0]
elif K.is_variable(spec[0]):
shape = (shape[0] * nb_params,) if len(shape) == 1 \
else shape[:-1] + (shape[-1] * nb_params,)
spec = K.concatenate(spec, axis=-1)
# ====== assign annotations ====== #
# only add role for trainable variables
for i in roles:
if isinstance(i, VariableRole) and K.is_trainable_variable(spec):
add_role(spec, i)
# return actual variable or expression
# override other parameters with same name
self._variables[name] = spec
# set parameter attribute for NNOps
setattr(nnops, name, spec)
return spec
def set_checkpoint(self, path=None, obj=None, variables=[],
increasing=True, max_checkpoint=-1,
save_history=None):
""" If `path` and `obj` given, the `obj` will be pickled
at `path` for every checkpoint, otherwise, store the
best values of `variables` in RAM
Parameters
----------
path: str
path to save the obj when the callback return save signal
obj: object
any pickle-able object you want to save
variables : {list of tensorflow.Variable}
external variables will be saved together with the
model
increasing : bool (default: True)
pass
max_checkpoint : int (default: 3)
pass
"""
self._save_path = path
self._save_obj = obj
if variables is not None:
variables = as_tuple(variables)
if len(variables) > 0:
from odin import backend as K
self._save_variables = [v for v in variables if K.is_variable(v)]
# other
self._checkpoint_increasing = bool(increasing)
self._checkpoint_max = int(max_checkpoint)
# ====== get the latest checkpoint count ====== #
if path is not None:
saved_files = []
base_dir = os.path.dirname(self._save_path)
base_name = os.path.basename(self._save_path)
pattern = re.compile(r"^%s\.\d+$" % re.escape(base_name))
for name in os.listdir(base_dir):
if not pattern.match(name):
continue
path = os.path.join(base_dir, name)
if self._save_path + '.' in path:
saved_files.append(path)
saved_files = sorted(saved_files,
key=lambda x: int(x.split('.')[-1]))
if len(saved_files) == 0:
self._current_checkpoint_count = 0
else:
self._current_checkpoint_count = int(saved_files[-1].split('.')[-1]) + 1
self._show_noti("[%s] Found lastest checkpoint: '.%d'" %
(ctext('MainLoop', 'red'), self._current_checkpoint_count))
# ====== history ====== #
if save_history is not None:
self._save_history = bool(save_history)
# save first checkpoint
if self._current_checkpoint_count == 0:
self._save(is_best=False)
def set_checkpoint(self, path=None, obj=None, variables=[],
increasing=True, max_checkpoint=-1,
save_history=None):
""" If `path` and `obj` given, the `obj` will be pickled
at `path` for every checkpoint, otherwise, store the
best values of `variables` in RAM
Parameters
----------
path: str
path to save the obj when the callback return save signal
obj: object
any pickle-able object you want to save
variables : {list of tensorflow.Variable}
external variables will be saved together with the
model
increasing : bool (default: True)
pass
max_checkpoint : int (default: 3)
pass
"""
self._save_path = path
self._save_obj = obj
if variables is not None:
variables = as_tuple(variables)
if len(variables) > 0:
from odin import backend as K
self._save_variables = [v for v in variables if K.is_variable(v)]
# other
self._checkpoint_increasing = bool(increasing)
self._checkpoint_max = int(max_checkpoint)
# ====== get the latest checkpoint count ====== #
if path is not None:
saved_files = []
base_dir = os.path.dirname(self._save_path)
base_name = os.path.basename(self._save_path)
pattern = re.compile(r"^%s\.\d+$" % re.escape(base_name))
for name in os.listdir(base_dir):
if not pattern.match(name):
continue
path = os.path.join(base_dir, name)
if self._save_path + '.' in path:
saved_files.append(path)
saved_files = sorted(saved_files,
key=lambda x: int(x.split('.')[-1]))
if len(saved_files) == 0:
self._current_checkpoint_count = 0
else:
self._current_checkpoint_count = int(saved_files[-1].split('.')[-1]) + 1
self._show_noti("[%s] Found lastest checkpoint: '.%d'" %
(ctext('MainLoop', 'red'), self._current_checkpoint_count))
# ====== history ====== #
if save_history is not None:
self._save_history = bool(save_history)
# save first checkpoint
if self._current_checkpoint_count == 0:
self._save(is_best=False)