def save_weights(self, filepath, format=None):
"""Input filepath, save model weights into a file of given format.
Use self.load_weights() to restore.
Parameters
----------
filepath : str
Filename to which the model weights will be saved.
format : str or None
Saved file format.
Value should be None, 'hdf5', 'npz', 'npz_dict' or 'ckpt'. Other format is not supported now.
1) If this is set to None, then the postfix of filepath will be used to decide saved format.
If the postfix is not in ['h5', 'hdf5', 'npz', 'ckpt'], then file will be saved in hdf5 format by default.
2) 'hdf5' will save model weights name in a list and each layer has its weights stored in a group of
the hdf5 file.
3) 'npz' will save model weights sequentially into a npz file.
4) 'npz_dict' will save model weights along with its name as a dict into a npz file.
5) 'ckpt' will save model weights into a tensorflow ckpt file.
Default None.
Examples
--------
1) Save model weights in hdf5 format by default.
>>> net = tl.models.vgg16()
>>> net.save_weights('./model.h5')
...
>>> net.load_weights('./model.h5')
2) Save model weights in npz/npz_dict format
>>> net = tl.models.vgg16()
>>> net.save_weights('./model.npz')
>>> net.save_weights('./model.npz', format='npz_dict')
"""
if self.all_weights is None or len(self.all_weights) == 0:
logging.warning("Model contains no weights or layers haven't been built, nothing will be saved")
return
if format is None:
postfix = filepath.split('.')[-1]
if postfix in ['h5', 'hdf5', 'npz', 'ckpt']:
format = postfix
else:
format = 'hdf5'
if format == 'hdf5' or format == 'h5':
utils.save_weights_to_hdf5(filepath, self)
elif format == 'npz':
utils.save_npz(self.all_weights, filepath)
elif format == 'npz_dict':
utils.save_npz_dict(self.all_weights, filepath)
elif format == 'ckpt':
# TODO: enable this when tf save ckpt is enabled
raise NotImplementedError("ckpt load/save is not supported now.")
else:
raise ValueError(
"Save format must be 'hdf5', 'npz', 'npz_dict' or 'ckpt'."
"Other format is not supported now."
)
def build(self, inputs):
if self.use_gemm:
raise Exception("TODO. The current version use tf.matmul for inferencing.")
if len(self.strides) != 2:
raise ValueError("len(strides) should be 2.")
try:
self.pre_channel = int(inputs.get_shape()[-1])
except Exception: # if pre_channel is ?, it happens when using Spatial Transformer Net
self.pre_channel = 1
logging.warning("[warnings] unknow input channels, set to 1")
self.shape = (self.filter_size[0], self.filter_size[1], self.pre_channel, self.n_filter)
self.strides = (1, self.strides[0], self.strides[1], 1)
self.W = tf.get_variable(
name=self.name+'\kernel', shape=self.shape, initializer=self.W_init, dtype=LayersConfig.tf_dtype, **self.W_init_args
)
if self.b_init:
self.b = tf.get_variable(
name=self.name+'\bias', shape=(self.shape[-1]), initializer=self.b_init, dtype=LayersConfig.tf_dtype,
**self.b_init_args
)
self.add_weights([self.W, self.b])
else:
self.add_weights(self.W)
def _check_mode(self, is_train):
"""Check whether this network is in a given mode.
Parameters
----------
is_train : boolean
Network's mode. True means training mode while False means evaluation mode.
Returns
-------
"""
# contradiction test
if is_train is None and self.is_train is None:
raise ValueError(
"Training / inference mode not defined. Argument `is_train` should be set as True / False. Otherwise please use `Model.train()` / `Model.eval()` to switch the mode."
)
elif is_train is not None and self.is_train is not None:
if is_train == self.is_train:
logging.warning(
"Training / inference mode redefined redundantly. Please EITHER use the argument `is_train` OR `Model.train()` / `Model.eval()` to define the mode."
)
else:
raise AttributeError(
"Training / inference mode mismatch. The argument `is_train` is set as %s, "
% is_train + "but the mode is currently set as %s. " %
('Training by Model.train()' if self.
is_train else 'Inference by Model.eval()') +
"Please EITHER use the argument `is_train` OR `Model.train()` / `Model.eval()` to define the mode."
)
def _check_mode(self, is_train):
"""Check whether this network is in a given mode.
Parameters
----------
is_train : boolean
Network's mode. True means training mode while False means evaluation mode.
Returns
-------
"""
# contradiction test
if is_train is None and self.is_train is None:
raise ValueError(
"Training / inference mode not defined. Argument `is_train` should be set as True / False. Otherwise please use `Model.train()` / `Model.eval()` to switch the mode."
)
elif is_train is not None and self.is_train is not None:
if is_train == self.is_train:
logging.warning(
"Training / inference mode redefined redundantly. Please EITHER use the argument `is_train` OR `Model.train()` / `Model.eval()` to define the mode."
)
else:
raise AttributeError(
"Training / inference mode mismatch. The argument `is_train` is set as %s, " % is_train +
"but the mode is currently set as %s. " %
('Training by Model.train()' if self.is_train else 'Inference by Model.eval()') +
"Please EITHER use the argument `is_train` OR `Model.train()` / `Model.eval()` to define the mode."
)
def __init__(
self,
prev_layer,
model_fn,
layer_args=None,
name='estimator_layer',
):
super(EstimatorLayer, self).__init__(prev_layer=prev_layer,
layer_args=layer_args,
name=name)
logging.info("EstimatorLayer %s: %s" % (self.name, model_fn))
if model_fn is None:
raise ValueError('model fn is None')
logging.warning(
"This API will be removed, please use LambdaLayer instead.")
with tf.variable_scope(name) as vs:
self.outputs = model_fn(self.inputs, **self.layer_args)
variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES,
scope=vs.name)
self._add_layers(self.outputs)
self._add_params(variables)
def save_weights(self, filepath, sess=None, format='hdf5'):
# TODO: Documentation pending
"""Input filepath and the session(optional), save model weights into a file of given format.
Use self.load_weights() to restore.
Parameters
----------
filepath : str
Filename to which the model weights will be saved.
sess : None or a tensorflow session
In eager mode, this should be left as None. In graph mode, must specify it with a tensorflow session.
format : Save file format
Value should be 'hdf5', 'npz', 'npz_dict' or 'ckpt'. Other format is not supported now.
'hdf5' will save model weights name in a list and each layer has its weights stored in a group of
the hdf5 file.
'npz' will save model weights sequentially into a npz file.
'npz_dict' will save model weights along with its name as a dict into a npz file.
'ckpt' will save model weights into a tensorflow ckpt file.
Examples
--------
1) Save model to hdf5 in eager mode
>>> net = tl.models.vgg.vgg16()
>>> net.save_weights('./model.h5')
2) Save model to npz in graph mode
>>> sess = tf.Session()
>>> sess.run(tf.global_variables_initializer())
>>> net.save_weights('./model.npz', sess=sess, format='npz')
Returns
-------
"""
if self.weights is None:
logging.warning(
"Model contains no weights or layers haven't been built, nothing will be saved"
)
return
if format == 'hdf5':
utils.save_weights_to_hdf5(filepath, self.weights, sess)
elif format == 'npz':
utils.save_npz(self.weights, filepath, sess)
elif format == 'npz_dict':
utils.save_npz_dict(self.weights, filepath, sess)
elif format == 'ckpt':
# TODO: enable this when tf save ckpt is enabled
raise NotImplementedError("ckpt load/save is not supported now.")
else:
raise ValueError(
"Save format must be 'hdf5', 'npz', 'npz_dict' or 'ckpt'."
"Other format is not supported now.")
def rename_kwargs(kwargs, aliases, end_support_version, func_name):
for alias, new in aliases.items():
if alias in kwargs:
if new in kwargs:
raise TypeError('{}() received both {} and {}'.format(func_name, alias, new))
warnings.warn('{}() - {} is deprecated; use {}'.format(func_name, alias, new), DeprecationWarning)
logging.warning(
"DeprecationWarning: {}(): "
"`{}` argument is deprecated and will be removed in version {}, "
"please change for `{}.`".format(func_name, alias, end_support_version, new)
)
kwargs[new] = kwargs.pop(alias)
def _check_mode(self, is_train):
# contradiction test
if is_train is None and self.is_train is None:
raise ValueError(
"Training / inference mode not defined. Argument `is_train` should be set as True / False. Otherwise please use `Model.train()` / `Model.eval()` to switch the mode."
)
elif is_train is not None and self.is_train is not None:
if is_train == self.is_train:
logging.warning(
"Training / inference mode redefined redundantly. Please EITHER use the argument `is_train` OR `Model.train()` / `Model.eval()` to define the mode."
)
else:
raise AttributeError(
"Training / inference mode mismatch. The argument `is_train` is set as %s, "
% is_train + "but the mode is currently set as %s. " %
('Training by Model.train()' if self.
is_train else 'Inference by Model.eval()') +
"Please EITHER use the argument `is_train` OR `Model.train()` / `Model.eval()` to define the mode."
)
def build(self, inputs):
if inputs.shape.as_list()[-1] is None:
logging.warning("unknown input channels, set to 1")
pre_channel = 1
self.shape = (self.filter_size[0], self.filter_size[1], pre_channel, self.n_filter)
self.strides = (1, strides[0], strides[1], 1)
self.W = tf.get_variable(
name=self.name+'\W_conv2d', shape=self.shape, initializer=self.W_init, dtype=LayersConfig.tf_dtype, **self.W_init_args
)
if self.b_init:
self.b = tf.get_variable(
name=self.name+'\b_conv2d', shape=(self.shape[-1]), initializer=self.b_init, dtype=LayersConfig.tf_dtype,
**self.b_init_args
)
if self.b_init:
self.add_weights([self.W, self.b])
else:
self.add_weights(self.W)
def __init__(
self,
prev_layer,
keras_layer,
keras_args=None,
name='keras_layer',
):
super(Keras, self).__init__(prev_layer=prev_layer,
keras_args=keras_args,
name=name)
logging.info("Keras %s: %s" % (self.name, keras_layer))
logging.warning(
"This API will be removed, please use LambdaLayer instead.")
with tf.variable_scope(name) as vs:
self.outputs = keras_layer(self.inputs, **self.keras_args)
variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES,
scope=vs.name)
self._add_layers(self.outputs)
self._add_params(variables)
def wrapper(wrapped, instance=None, args=None, kwargs=None):
validate_deprecation_args(date, instructions)
if _PRINT_DEPRECATION_WARNINGS:
class_or_func_name = get_qualified_name(wrapped)
if class_or_func_name not in _PRINTED_WARNING:
if warn_once:
_PRINTED_WARNING[class_or_func_name] = True
from tensorlayer import logging
logging.warning(
'%s: `%s.%s` (in file: %s) is deprecated and will be removed %s.\n'
'Instructions for updating: %s\n' %
("Class" if inspect.isclass(wrapped) else "Function",
wrapped.__module__, class_or_func_name,
wrapped.__code__.co_filename,
'in a future version' if date is None else
('after %s' % date), instructions))
return wrapped(*args, **kwargs)
def __init__(
self,
prev_layer,
n_filter=32,
filter_size=(3, 3),
strides=(1, 1),
padding='SAME',
act=None,
bitW=8,
bitA=8,
use_gemm=False,
W_init=tf.truncated_normal_initializer(stddev=0.02),
b_init=tf.constant_initializer(value=0.0),
W_init_args=None,
b_init_args=None,
use_cudnn_on_gpu=None,
data_format=None,
name='quan_cnn2d',
):
super(RTQuanConv2d, self).__init__(prev_layer=prev_layer,
act=act,
W_init_args=W_init_args,
b_init_args=b_init_args,
name=name)
logging.info(
"RTQuanConv2d %s: n_filter: %d filter_size: %s strides: %s pad: %s act: %s"
% (self.name, n_filter, str(filter_size), str(strides), padding,
self.act.__name__ if self.act is not None else 'No Activation'))
if use_gemm:
raise Exception(
"TODO. The current version use tf.matmul for inferencing.")
if len(strides) != 2:
raise ValueError("len(strides) should be 2.")
try:
pre_channel = int(prev_layer.outputs.get_shape()[-1])
except Exception: # if pre_channel is ?, it happens when using Spatial Transformer Net
pre_channel = 1
logging.warning("[warnings] unknow input channels, set to 1")
shape = (filter_size[0], filter_size[1], pre_channel, n_filter)
strides = (1, strides[0], strides[1], 1)
with tf.variable_scope(name):
W = tf.get_variable(name='W_conv2d',
shape=shape,
initializer=W_init,
dtype=LayersConfig.tf_dtype,
**self.W_init_args)
self.outputs = tf.nn.conv2d(self.inputs,
W,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
scale = tf.get_variable(name='W_scale',
shape=n_filter,
initializer=W_init,
dtype=LayersConfig.tf_dtype,
**self.W_init_args)
output_channel = self.outputs.get_shape()
new_scale = tf.tile(
scale,
[output_channel[0] * output_channel[1] * output_channel[2]])
new_scale = tf.reshape(new_scale, output_channel)
self.outputs = tf.divide(self.outputs, new_scale)
if b_init:
b = tf.get_variable(name='b_conv2d',
shape=(shape[-1]),
initializer=b_init,
dtype=LayersConfig.tf_dtype,
**self.b_init_args)
self.outputs = tf.nn.bias_add(self.outputs, b, name='bias_add')
self.outputs = self._apply_activation(self.outputs)
self._add_layers(self.outputs)
if b_init:
self._add_params([W, b, scale])
else:
self._add_params([W, scale])
def __init__(
self,
prev_layer,
cell_fn,
cell_init_args=None,
n_hidden=100,
initializer=tf.compat.v1.initializers.random_uniform(-0.1, 0.1),
n_steps=5,
fw_initial_state=None,
bw_initial_state=None,
dropout=None,
n_layer=1,
return_last=False,
return_seq_2d=False,
name='birnn',
):
super(BiRNN, self).__init__(prev_layer=prev_layer,
cell_init_args=cell_init_args,
name=name)
if self.cell_init_args:
self.cell_init_args[
'state_is_tuple'] = True # 'use_peepholes': True,
if 'GRU' in cell_fn.__name__:
try:
self.cell_init_args.pop('state_is_tuple')
except Exception:
logging.warning("pop state_is_tuple fails.")
if cell_fn is None:
raise Exception("Please put in cell_fn")
logging.info(
"BiRNN %s: n_hidden: %d n_steps: %d in_dim: %d in_shape: %s cell_fn: %s dropout: %s n_layer: %d "
% (self.name, n_hidden, n_steps, self.inputs.get_shape().ndims,
self.inputs.get_shape(), cell_fn.__name__, dropout, n_layer))
fixed_batch_size = self.inputs.get_shape().with_rank_at_least(1)[0]
if fixed_batch_size.value:
self.batch_size = fixed_batch_size.value
logging.info(" RNN batch_size (concurrent processes): %d" %
self.batch_size)
else:
self.batch_size = array_ops.shape(self.inputs)[0]
logging.info(
" non specified batch_size, uses a tensor instead.")
# Input dimension should be rank 3 [batch_size, n_steps(max), n_features]
try:
self.inputs.get_shape().with_rank(3)
except Exception:
raise Exception(
"RNN : Input dimension should be rank 3 : [batch_size, n_steps, n_features]"
)
with tf.compat.v1.variable_scope(name, initializer=initializer) as vs:
rnn_creator = lambda: cell_fn(num_units=n_hidden,
**self.cell_init_args)
# Apply dropout
if dropout:
if isinstance(
dropout,
(tuple, list)): # type(dropout) in [tuple, list]:
in_keep_prob = dropout[0]
out_keep_prob = dropout[1]
elif isinstance(dropout, float):
in_keep_prob, out_keep_prob = dropout, dropout
else:
raise Exception(
"Invalid dropout type (must be a 2-D tuple of "
"float)")
DropoutWrapper_fn = tf.contrib.rnn.DropoutWrapper
cell_creator = lambda is_last=True: DropoutWrapper_fn(
rnn_creator(),
input_keep_prob=in_keep_prob,
output_keep_prob=out_keep_prob if is_last else 1.0)
else:
cell_creator = rnn_creator
self.fw_cell = cell_creator()
self.bw_cell = cell_creator()
# Apply multiple layers
if n_layer > 1:
MultiRNNCell_fn = tf.contrib.rnn.MultiRNNCell
if dropout:
try:
self.fw_cell = MultiRNNCell_fn([
cell_creator(is_last=i == n_layer - 1)
for i in range(n_layer)
],
state_is_tuple=True)
self.bw_cell = MultiRNNCell_fn([
cell_creator(is_last=i == n_layer - 1)
for i in range(n_layer)
],
state_is_tuple=True)
except Exception:
self.fw_cell = MultiRNNCell_fn([
cell_creator(is_last=i == n_layer - 1)
for i in range(n_layer)
])
self.bw_cell = MultiRNNCell_fn([
cell_creator(is_last=i == n_layer - 1)
for i in range(n_layer)
])
else:
try:
self.fw_cell = MultiRNNCell_fn(
[cell_creator() for _ in range(n_layer)],
state_is_tuple=True)
self.bw_cell = MultiRNNCell_fn(
[cell_creator() for _ in range(n_layer)],
state_is_tuple=True)
except Exception:
self.fw_cell = MultiRNNCell_fn(
[cell_creator() for _ in range(n_layer)])
self.bw_cell = MultiRNNCell_fn(
[cell_creator() for _ in range(n_layer)])
# Initial state of RNN
if fw_initial_state is None:
self.fw_initial_state = self.fw_cell.zero_state(
self.batch_size,
dtype=LayersConfig.tf_dtype) # dtype=tf.float32)
else:
self.fw_initial_state = fw_initial_state
if bw_initial_state is None:
self.bw_initial_state = self.bw_cell.zero_state(
self.batch_size,
dtype=LayersConfig.tf_dtype) # dtype=tf.float32)
else:
self.bw_initial_state = bw_initial_state
# exit()
# Feedforward to MultiRNNCell
list_rnn_inputs = tf.unstack(self.inputs, axis=1)
bidirectional_rnn_fn = tf.contrib.rnn.static_bidirectional_rnn
outputs, fw_state, bw_state = bidirectional_rnn_fn( # outputs, fw_state, bw_state = tf.contrib.rnn.static_bidirectional_rnn(
cell_fw=self.fw_cell,
cell_bw=self.bw_cell,
inputs=list_rnn_inputs,
initial_state_fw=self.fw_initial_state,
initial_state_bw=self.bw_initial_state)
if return_last:
raise Exception("Do not support return_last at the moment.")
# self.outputs = outputs[-1]
else:
self.outputs = outputs
if return_seq_2d:
# 2D Tensor [n_example, n_hidden]
self.outputs = tf.reshape(tf.concat(outputs, 1),
[-1, n_hidden * 2])
else:
# <akara>: stack more RNN layer after that
# 3D Tensor [n_example/n_steps, n_steps, n_hidden]
self.outputs = tf.reshape(tf.concat(outputs, 1),
[-1, n_steps, n_hidden * 2])
self.fw_final_state = fw_state
self.bw_final_state = bw_state
# Retrieve just the RNN variables.
rnn_variables = tf.compat.v1.get_collection(TF_GRAPHKEYS_VARIABLES,
scope=vs.name)
logging.info(" n_params : %d" % (len(rnn_variables)))
self._add_layers(self.outputs)
self._add_params(rnn_variables)
def __init__(
self,
net_encode_in,
net_decode_in,
cell_fn, #tf.nn.rnn_cell.LSTMCell,
cell_init_args=None,
n_hidden=256,
initializer=tf.compat.v1.initializers.random_uniform(-0.1, 0.1),
encode_sequence_length=None,
decode_sequence_length=None,
initial_state_encode=None,
initial_state_decode=None,
dropout=None,
n_layer=1,
return_seq_2d=False,
name='seq2seq',
):
super(Seq2Seq,
self).__init__(prev_layer=[net_encode_in, net_decode_in],
cell_init_args=cell_init_args,
name=name)
if self.cell_init_args:
self.cell_init_args[
'state_is_tuple'] = True # 'use_peepholes': True,
if cell_fn is None:
raise ValueError("cell_fn cannot be set to None")
if 'GRU' in cell_fn.__name__:
try:
cell_init_args.pop('state_is_tuple')
except Exception:
logging.warning("pop state_is_tuple fails.")
logging.info(
"[*] Seq2Seq %s: n_hidden: %d cell_fn: %s dropout: %s n_layer: %d"
% (self.name, n_hidden, cell_fn.__name__, dropout, n_layer))
with tf.compat.v1.variable_scope(name):
# tl.layers.set_name_reuse(reuse)
# network = InputLayer(self.inputs, name=name+'/input')
network_encode = DynamicRNN(net_encode_in,
cell_fn=cell_fn,
cell_init_args=self.cell_init_args,
n_hidden=n_hidden,
initializer=initializer,
initial_state=initial_state_encode,
dropout=dropout,
n_layer=n_layer,
sequence_length=encode_sequence_length,
return_last=False,
return_seq_2d=True,
name='encode')
# vs.reuse_variables()
# tl.layers.set_name_reuse(True)
network_decode = DynamicRNN(
net_decode_in,
cell_fn=cell_fn,
cell_init_args=self.cell_init_args,
n_hidden=n_hidden,
initializer=initializer,
initial_state=(network_encode.final_state
if initial_state_decode is None else
initial_state_decode),
dropout=dropout,
n_layer=n_layer,
sequence_length=decode_sequence_length,
return_last=False,
return_seq_2d=return_seq_2d,
name='decode')
self.outputs = network_decode.outputs
# rnn_variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES, scope=vs.name)
# Initial state
self.initial_state_encode = network_encode.initial_state
self.initial_state_decode = network_decode.initial_state
# Final state
self.final_state_encode = network_encode.final_state
self.final_state_decode = network_decode.final_state
# self.sequence_length = sequence_length
self._add_layers(network_encode.all_layers)
self._add_params(network_encode.all_params)
self._add_dropout_layers(network_encode.all_drop)
self._add_layers(network_decode.all_layers)
self._add_params(network_decode.all_params)
self._add_dropout_layers(network_decode.all_drop)
self._add_layers(self.outputs)
def set_name_reuse(enable=True):
logging.warning('this method is DEPRECATED and has no effect, please remove it from your code.')
def save_weights(self, filepath, format=None):
"""Input filepath and the session(optional), save model weights into a file of given format.
Use self.load_weights() to restore.
Parameters
----------
filepath : str
Filename to which the model weights will be saved.
format : str or None
Saved file format.
Value should be None, 'hdf5', 'npz', 'npz_dict' or 'ckpt'. Other format is not supported now.
1) If this is set to None, then the postfix of filepath will be used to decide saved format.
If the postfix is not in ['h5', 'hdf5', 'npz', 'ckpt'], then file will be saved in hdf5 format by default.
2) 'hdf5' will save model weights name in a list and each layer has its weights stored in a group of
the hdf5 file.
3) 'npz' will save model weights sequentially into a npz file.
4) 'npz_dict' will save model weights along with its name as a dict into a npz file.
5) 'ckpt' will save model weights into a tensorflow ckpt file.
Default None.
Examples
--------
1) Save model weights in hdf5 format by default.
>>> net = tl.models.vgg16()
>>> net.save_weights('./model.h5')
...
>>> net.load_weights('./model.h5')
2) Save model weights in npz/npz_dict format
>>> net = tl.models.vgg16()
>>> net.save_weights('./model.npz')
>>> net.save_weights('./model.npz', format='npz_dict')
Returns
-------
"""
if self.weights is None or len(self.weights) == 0:
logging.warning("Model contains no weights or layers haven't been built, nothing will be saved")
return
if format is None:
postfix = filepath.split('.')[-1]
if postfix in ['h5', 'hdf5', 'npz', 'ckpt']:
format = postfix
else:
format = 'hdf5'
if format == 'hdf5' or format == 'h5':
utils.save_weights_to_hdf5(filepath, self)
elif format == 'npz':
utils.save_npz(self.weights, filepath)
elif format == 'npz_dict':
utils.save_npz_dict(self.weights, filepath)
elif format == 'ckpt':
# TODO: enable this when tf save ckpt is enabled
raise NotImplementedError("ckpt load/save is not supported now.")
else:
raise ValueError(
"Save format must be 'hdf5', 'npz', 'npz_dict' or 'ckpt'."
"Other format is not supported now."
)
def __init__(
self,
prev_layer,
n_filter=32,
filter_size=(3, 3),
strides=(1, 1),
act=None,
padding='SAME',
use_gemm=False,
W_init=tf.truncated_normal_initializer(stddev=0.02),
b_init=tf.constant_initializer(value=0.0),
W_init_args=None,
b_init_args=None,
use_cudnn_on_gpu=None,
data_format=None,
# act=None,
# shape=(5, 5, 1, 100),
# strides=(1, 1, 1, 1),
# padding='SAME',
# W_init=tf.truncated_normal_initializer(stddev=0.02),
# b_init=tf.constant_initializer(value=0.0),
# W_init_args=None,
# b_init_args=None,
# use_cudnn_on_gpu=None,
# data_format=None,
name='ternary_cnn2d',
):
super(TernaryConv2d, self
).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, b_init_args=b_init_args, name=name)
logging.info(
"TernaryConv2d %s: n_filter: %d filter_size: %s strides: %s pad: %s act: %s" % (
self.name, n_filter, str(filter_size), str(strides), padding,
self.act.__name__ if self.act is not None else 'No Activation'
)
)
if len(strides) != 2:
raise ValueError("len(strides) should be 2.")
if use_gemm:
raise Exception("TODO. The current version use tf.matmul for inferencing.")
try:
pre_channel = int(prev_layer.outputs.get_shape()[-1])
except Exception: # if pre_channel is ?, it happens when using Spatial Transformer Net
pre_channel = 1
logging.warning("unknow input channels, set to 1")
shape = (filter_size[0], filter_size[1], pre_channel, n_filter)
strides = (1, strides[0], strides[1], 1)
with tf.variable_scope(name):
W = tf.get_variable(
name='W_conv2d', shape=shape, initializer=W_init, dtype=LayersConfig.tf_dtype, **self.W_init_args
)
alpha = compute_alpha(W)
W = ternary_operation(W)
W = tf.multiply(alpha, W)
self.outputs = tf.nn.conv2d(
self.inputs, W, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format
)
if b_init:
b = tf.get_variable(
name='b_conv2d', shape=(shape[-1]), initializer=b_init, dtype=LayersConfig.tf_dtype,
**self.b_init_args
)
self.outputs = tf.nn.bias_add(self.outputs, b, name='bias_add')
self.outputs = self._apply_activation(self.outputs)
self._add_layers(self.outputs)
if b_init:
self._add_params([W, b])
else:
self._add_params(W)
def __init__(
self,
prev_layer,
n_filter=32,
filter_size=(3, 3),
strides=(1, 1),
padding='SAME',
act=None,
decay=0.9,
epsilon=1e-5,
is_train=False,
gamma_init=tf.compat.v1.initializers.ones,
beta_init=tf.compat.v1.initializers.zeros,
bitW=8,
bitA=8,
use_gemm=False,
W_init=tf.compat.v1.initializers.truncated_normal(stddev=0.02),
W_init_args=None,
use_cudnn_on_gpu=None,
data_format=None,
name='quan_cnn2d_bn',
):
super(QuanConv2dWithBN, self).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, name=name)
logging.info(
"QuanConv2dWithBN %s: n_filter: %d filter_size: %s strides: %s pad: %s act: %s " % (
self.name, n_filter, filter_size, str(strides), padding,
self.act.__name__ if self.act is not None else 'No Activation'
)
)
x = self.inputs
self.inputs = quantize_active_overflow(self.inputs, bitA) # Do not remove
if use_gemm:
raise Exception("TODO. The current version use tf.matmul for inferencing.")
if len(strides) != 2:
raise ValueError("len(strides) should be 2.")
try:
pre_channel = int(prev_layer.outputs.get_shape()[-1])
except Exception: # if pre_channel is ?, it happens when using Spatial Transformer Net
pre_channel = 1
logging.warning("[warnings] unknow input channels, set to 1")
shape = (filter_size[0], filter_size[1], pre_channel, n_filter)
strides = (1, strides[0], strides[1], 1)
with tf.compat.v1.variable_scope(name):
W = tf.compat.v1.get_variable(
name='W_conv2d', shape=shape, initializer=W_init, dtype=LayersConfig.tf_dtype, **self.W_init_args
)
conv = tf.nn.conv2d(
x, W, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format
)
para_bn_shape = conv.get_shape()[-1:]
if gamma_init:
scale_para = tf.compat.v1.get_variable(
name='scale_para', shape=para_bn_shape, initializer=gamma_init, dtype=LayersConfig.tf_dtype,
trainable=is_train
)
else:
scale_para = None
if beta_init:
offset_para = tf.compat.v1.get_variable(
name='offset_para', shape=para_bn_shape, initializer=beta_init, dtype=LayersConfig.tf_dtype,
trainable=is_train
)
else:
offset_para = None
moving_mean = tf.compat.v1.get_variable(
'moving_mean', para_bn_shape, initializer=tf.compat.v1.initializers.constant(1.),
dtype=LayersConfig.tf_dtype, trainable=False
)
moving_variance = tf.compat.v1.get_variable(
'moving_variance',
para_bn_shape,
initializer=tf.compat.v1.initializers.constant(1.),
dtype=LayersConfig.tf_dtype,
trainable=False,
)
mean, variance = tf.nn.moments(x=conv, axes=list(range(len(conv.get_shape()) - 1)))
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean, decay, zero_debias=False
) # if zero_debias=True, has bias
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance, decay, zero_debias=False
) # if zero_debias=True, has bias
def mean_var_with_update():
with tf.control_dependencies([update_moving_mean, update_moving_variance]):
return tf.identity(mean), tf.identity(variance)
if is_train:
mean, var = mean_var_with_update()
else:
mean, var = moving_mean, moving_variance
w_fold = _w_fold(W, scale_para, var, epsilon)
bias_fold = _bias_fold(offset_para, scale_para, mean, var, epsilon)
W = quantize_weight_overflow(w_fold, bitW)
conv_fold = tf.nn.conv2d(
self.inputs, W, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format
)
self.outputs = tf.nn.bias_add(conv_fold, bias_fold, name='bn_bias_add')
self.outputs = self._apply_activation(self.outputs)
self._add_layers(self.outputs)
self._add_params([W, scale_para, offset_para, moving_mean, moving_variance])
def __init__(
self,
net_encode_in,
net_decode_in,
cell_fn, #tf.nn.rnn_cell.LSTMCell,
cell_init_args=None,
n_hidden=256,
initializer=tf.compat.v1.initializers.random_uniform(-0.1, 0.1),
encode_sequence_length=None,
decode_sequence_length=None,
initial_state_encode=None,
initial_state_decode=None,
dropout=None,
n_layer=1,
return_seq_2d=False,
name='seq2seq',
):
super(Seq2Seq,
self).__init__(prev_layer=[net_encode_in, net_decode_in], cell_init_args=cell_init_args, name=name)
if self.cell_init_args:
self.cell_init_args['state_is_tuple'] = True # 'use_peepholes': True,
if cell_fn is None:
raise ValueError("cell_fn cannot be set to None")
if 'GRU' in cell_fn.__name__:
try:
cell_init_args.pop('state_is_tuple')
except Exception:
logging.warning("pop state_is_tuple fails.")
logging.info(
"[*] Seq2Seq %s: n_hidden: %d cell_fn: %s dropout: %s n_layer: %d" %
(self.name, n_hidden, cell_fn.__name__, dropout, n_layer)
)
with tf.compat.v1.variable_scope(name):
# tl.layers.set_name_reuse(reuse)
# network = InputLayer(self.inputs, name=name+'/input')
network_encode = DynamicRNN(
net_encode_in, cell_fn=cell_fn, cell_init_args=self.cell_init_args, n_hidden=n_hidden,
initializer=initializer, initial_state=initial_state_encode, dropout=dropout, n_layer=n_layer,
sequence_length=encode_sequence_length, return_last=False, return_seq_2d=True, name='encode'
)
# vs.reuse_variables()
# tl.layers.set_name_reuse(True)
network_decode = DynamicRNN(
net_decode_in, cell_fn=cell_fn, cell_init_args=self.cell_init_args, n_hidden=n_hidden,
initializer=initializer,
initial_state=(network_encode.final_state if initial_state_decode is None else initial_state_decode),
dropout=dropout, n_layer=n_layer, sequence_length=decode_sequence_length, return_last=False,
return_seq_2d=return_seq_2d, name='decode'
)
self.outputs = network_decode.outputs
# rnn_variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES, scope=vs.name)
# Initial state
self.initial_state_encode = network_encode.initial_state
self.initial_state_decode = network_decode.initial_state
# Final state
self.final_state_encode = network_encode.final_state
self.final_state_decode = network_decode.final_state
# self.sequence_length = sequence_length
self._add_layers(network_encode.all_layers)
self._add_params(network_encode.all_params)
self._add_dropout_layers(network_encode.all_drop)
self._add_layers(network_decode.all_layers)
self._add_params(network_decode.all_params)
self._add_dropout_layers(network_decode.all_drop)
self._add_layers(self.outputs)
def set_name_reuse(enable=True):
logging.warning(
'this method is DEPRECATED and has no effect, please remove it from your code.'
)
def clear_layers_name():
logging.warning(
'this method is DEPRECATED and has no effect, please remove it from your code.'
)
def __init__(self,
prev_layer,
n_filter=32,
filter_size=(3, 3),
strides=(1, 1),
act=None,
padding='SAME',
use_gemm=False,
W_init=None,
b_init=None,
W_init_args=None,
b_init_args=None,
use_cudnn_on_gpu=None,
data_format=None,
name='my_binary_conv2d'):
super(MyBinaryConv2d, self).__init__(prev_layer=prev_layer,
act=act,
W_init_args=W_init_args,
b_init_args=b_init_args,
name=name)
logging.info(
"MyBinaryConv2d %s: n_filter: %d filter_size: %s strides: %s pad: %s act: %s"
% (self.name, n_filter, str(filter_size), str(strides), padding,
self.act.__name__ if self.act is not None else 'No Activation'))
if use_gemm:
raise Exception(
"TODO. The current version use tf.matmul for inferencing.")
try:
pre_channel = int(prev_layer.outputs.get_shape()[-1])
except Exception:
pre_channel = 1
logging.warning("unknown input channels, set to 1")
w_shape = (filter_size[0], filter_size[1], pre_channel, n_filter)
strides = (1, strides[0], strides[1], 1)
with tf.variable_scope(name):
W = tf.get_variable(name='binary_w',
shape=w_shape,
initializer=W_init,
dtype=LayersConfig.tf_dtype,
**self.W_init_args)
binary_W = binary_weight(W)
self.outputs = tf.nn.conv2d(self.inputs, binary_W, strides,
padding, use_cudnn_on_gpu, data_format)
if b_init:
b = tf.get_variable(name='bias',
shape=(w_shape[-1]),
initializer=b_init,
dtype=LayersConfig.tf_dtype,
**self.b_init_args)
self.outputs = tf.nn.bias_add(self.outputs, b, name='b_add')
self.outputs = self._apply_activation(self.outputs)
self._add_layers(self.outputs)
if b_init:
self._add_params([W, b])
else:
self._add_params(W)
