def dnn(x):
with tf.variable_scope('Layer1'):
# Creating variable using TFLearn
W1 = va.variable(name='W', shape=[len(X_train[0]), first_layer],
initializer='uniform_scaling',
regularizer='L2')
b1 = va.variable(name='b', shape=[first_layer])
x = tf.nn.tanh(tf.add(tf.matmul(x, W1), b1))
with tf.variable_scope('Layer2'):
W2 = va.variable(name='W', shape=[first_layer, second_layer],
initializer='uniform_scaling',
regularizer='L2')
b2 = va.variable(name='b', shape=[second_layer])
x = tf.nn.tanh(tf.add(tf.matmul(x, W2), b2))
with tf.variable_scope('Layer3'):
W3 = va.variable(name='W', shape=[second_layer, third_layer],
initializer='uniform_scaling',regularizer='L2')
b3 = va.variable(name='b', shape=[third_layer])
x = tf.add(tf.matmul(x, W3), b3)
with tf.variable_scope('Layer4'):
W4 = va.variable(name='W', shape=[third_layer, len(Y_train[0])],
initializer='uniform_scaling',regularizer='L2')
b4 = va.variable(name='b', shape=[len(Y_train[0])])
x = tf.add(tf.matmul(x, W4), b4)
return x,W4,b4
def dnn(x):
with tf.variable_scope('Layer1'):
# Creating variable using TFLearn
W1 = va.variable(name='W',
shape=[784, 256],
initializer='uniform_scaling',
regularizer='L2')
b1 = va.variable(name='b', shape=[256])
x = tf.nn.tanh(tf.add(tf.matmul(x, W1), b1))
with tf.variable_scope('Layer2'):
W2 = va.variable(name='W',
shape=[256, 256],
initializer='uniform_scaling',
regularizer='L2')
b2 = va.variable(name='b', shape=[256])
x = tf.nn.tanh(tf.add(tf.matmul(x, W2), b2))
with tf.variable_scope('Layer3'):
W3 = va.variable(name='W',
shape=[256, 10],
initializer='uniform_scaling')
b3 = va.variable(name='b', shape=[10])
x = tf.add(tf.matmul(x, W3), b3)
return x
def dnn(x):
with tf.variable_scope('Layer1'):
# Creating variable using TFLearn
W1 = va.variable(name='W', shape=[784, 256],
initializer='uniform_scaling',
regularizer='L2')
b1 = va.variable(name='b', shape=[256])
x = tf.nn.tanh(tf.add(tf.matmul(x, W1), b1))
with tf.variable_scope('Layer2'):
W2 = va.variable(name='W', shape=[256, 256],
initializer='uniform_scaling',
regularizer='L2')
b2 = va.variable(name='b', shape=[256])
x = tf.nn.tanh(tf.add(tf.matmul(x, W2), b2))
with tf.variable_scope('Layer3'):
W3 = va.variable(name='W', shape=[256, 10],
initializer='uniform_scaling')
b3 = va.variable(name='b', shape=[10])
x = tf.add(tf.matmul(x, W3), b3)
return x
def single_unit(incoming, activation='linear', bias=True, trainable=True,
restore=True, reuse=False, scope=None, name="Linear"):
""" Single Unit.
A single unit (Linear) Layer.
Input:
1-D Tensor [samples]. If not 2D, input will be flatten.
Output:
1-D Tensor [samples].
Arguments:
incoming: `Tensor`. Incoming Tensor.
activation: `str` (name) or `function`. Activation applied to this
layer (see tflearn.activations). Default: 'linear'.
bias: `bool`. If True, a bias is used.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model.
reuse: `bool`. If True and 'scope' is provided, this layer variables
will be reused (shared).
scope: `str`. Define this layer scope (optional). A scope can be
used to share variables between layers. Note that scope will
override name.
name: A name for this layer (optional). Default: 'Linear'.
Attributes:
W: `Tensor`. Variable representing weight.
b: `Tensor`. Variable representing bias.
"""
input_shape = utils.get_incoming_shape(incoming)
n_inputs = int(np.prod(input_shape[1:]))
# Build variables and inference.
with tf.variable_op_scope([incoming], scope, name, reuse=reuse) as scope:
name = scope.name
W = va.variable('W', shape=[n_inputs],
initializer=tf.constant_initializer(np.random.randn()),
trainable=trainable, restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, W)
b = None
if bias:
b = va.variable('b', shape=[n_inputs],
initializer=tf.constant_initializer(np.random.randn()),
trainable=trainable, restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 1:
inference = tf.reshape(inference, [-1])
inference = tf.mul(inference, W)
if b: inference = tf.add(inference, b)
if isinstance(activation, str):
inference = activations.get(activation)(inference)
elif hasattr(activation, '__call__'):
inference = activation(inference)
else:
raise ValueError("Invalid Activation.")
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, inference)
# Add attributes to Tensor to easy access weights.
inference.scope = scope
inference.W = W
inference.b = b
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def fully_connected(incoming,
n_units,
activation='linear',
bias=True,
weights_init='truncated_normal',
bias_init='zeros',
regularizer=None,
weight_decay=0.001,
trainable=True,
restore=True,
name="FullyConnected"):
""" Fully Connected.
A fully connected layer.
Input:
(2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Output:
2D Tensor [samples, n_units].
Arguments:
incoming: `Tensor`. Incoming (2+)D Tensor.
n_units: `int`, number of units for this layer.
activation: `str` (name) or `Tensor`. Activation applied to this layer.
(see tflearn.activations). Default: 'linear'.
bias: `bool`. If True, a bias is used.
weights_init: `str` (name) or `Tensor`. Weights initialization.
(see tflearn.initializations) Default: 'truncated_normal'.
bias_init: `str` (name) or `Tensor`. Bias initialization.
(see tflearn.initializations) Default: 'zeros'.
regularizer: `str` (name) or `Tensor`. Add a regularizer to this
layer weights (see tflearn.regularizers). Default: None.
weight_decay: `float`. Regularizer decay parameter. Default: 0.001.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model
name: A name for this layer (optional). Default: 'FullyConnected'.
Attributes:
scope: `Scope`. This layer scope.
W: `Tensor`. Variable representing units weights.
b: `Tensor`. Variable representing biases.
"""
input_shape = utils.get_incoming_shape(incoming)
n_inputs = int(np.prod(input_shape[1:]))
# Build variables and inference.
with tf.name_scope(name) as scope:
W_init = initializations.get(weights_init)()
W_regul = None
if regularizer:
W_regul = lambda x: losses.get(regularizer)(x, weight_decay)
W = va.variable(scope + 'W',
shape=[n_inputs, n_units],
regularizer=W_regul,
initializer=W_init,
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope, W)
b = None
if bias:
b_init = initializations.get(bias_init)()
b = va.variable(scope + 'b',
shape=[n_units],
initializer=b_init,
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope, b)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 2:
inference = tf.reshape(inference, [-1, n_inputs])
inference = tf.matmul(inference, W)
if b: inference = tf.nn.bias_add(inference, b)
inference = activations.get(activation)(inference)
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, inference)
# Add attributes to Tensor to easy access weights.
inference.scope = scope
inference.W = W
inference.b = b
return inference
def single_unit(incoming,
activation='linear',
bias=True,
trainable=True,
restore=True,
name="Linear"):
""" Single Unit.
A single unit (Linear) Layer.
Input:
1-D Tensor [samples]. If not 2D, input will be flatten.
Output:
1-D Tensor [samples].
Arguments:
incoming: `Tensor`. Incoming Tensor.
activation: `str` (name) or `Tensor`. Activation applied to this layer.
(see tflearn.activations). Default: 'linear'.
bias: `bool`. If True, a bias is used.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model.
name: A name for this layer (optional). Default: 'Dense'.
Attributes:
W: `Tensor`. Variable representing weight.
b: `Tensor`. Variable representing bias.
"""
input_shape = utils.get_incoming_shape(incoming)
n_inputs = int(np.prod(input_shape[1:]))
# Build variables and inference.
with tf.name_scope(name) as scope:
W = va.variable(scope + 'W',
shape=[n_inputs],
initializer=tf.constant_initializer(np.random.randn()),
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope, W)
b = None
if bias:
b = va.variable(scope + 'b',
shape=[n_inputs],
initializer=tf.constant_initializer(
np.random.randn()),
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope, b)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 1:
inference = tf.reshape(inference, [-1])
inference = tf.mul(inference, W)
if b: inference = tf.add(inference, b)
inference = activations.get(activation)(inference)
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, inference)
# Add attributes to Tensor to easy access weights.
inference.scope = scope
inference.W = W
inference.b = b
return inference
def fully_connected(incoming, n_units, activation='linear', bias=True,
weights_init='truncated_normal', bias_init='zeros',
regularizer=None, weight_decay=0.001, trainable=True,
restore=True, name="FullyConnected"):
""" Fully Connected.
A fully connected layer.
Input:
(2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Output:
2D Tensor [samples, n_units].
Arguments:
incoming: `Tensor`. Incoming (2+)D Tensor.
n_units: `int`, number of units for this layer.
activation: `str` (name) or `Tensor`. Activation applied to this layer.
(see tflearn.activations). Default: 'linear'.
bias: `bool`. If True, a bias is used.
weights_init: `str` (name) or `Tensor`. Weights initialization.
(see tflearn.initializations) Default: 'truncated_normal'.
bias_init: `str` (name) or `Tensor`. Bias initialization.
(see tflearn.initializations) Default: 'zeros'.
regularizer: `str` (name) or `Tensor`. Add a regularizer to this
layer weights (see tflearn.regularizers). Default: None.
weight_decay: `float`. Regularizer decay parameter. Default: 0.001.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model
name: A name for this layer (optional). Default: 'FullyConnected'.
Attributes:
scope: `Scope`. This layer scope.
W: `Tensor`. Variable representing units weights.
b: `Tensor`. Variable representing biases.
"""
input_shape = utils.get_incoming_shape(incoming)
n_inputs = int(np.prod(input_shape[1:]))
# Build variables and inference.
with tf.name_scope(name) as scope:
W_init = weights_init
if isinstance(weights_init, str):
W_init = initializations.get(weights_init)()
W_regul = None
if regularizer:
W_regul = lambda x: losses.get(regularizer)(x, weight_decay)
W = va.variable(scope + 'W', shape=[n_inputs, n_units],
regularizer=W_regul, initializer=W_init,
trainable=trainable, restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope, W)
b = None
if bias:
b_init = initializations.get(bias_init)()
b = va.variable(scope + 'b', shape=[n_units],
initializer=b_init, trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope, b)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 2:
inference = tf.reshape(inference, [-1, n_inputs])
inference = tf.matmul(inference, W)
if b: inference = tf.nn.bias_add(inference, b)
inference = activations.get(activation)(inference)
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, inference)
# Add attributes to Tensor to easy access weights.
inference.scope = scope
inference.W = W
inference.b = b
return inference
weights_init="normal") # hidden layer of size 2
output_layer = fully_connected(hidden_layer,
1,
activation='linear',
name="outputLayer_Weights",
weights_init="normal") # output layer of size 1
# Hyper parameters for the one class Neural Network
v = 0.04
nu = 0.04
# Initialize rho
value = 0.0001
init = tf.constant_initializer(value)
rho = va.variable(name='rho', dtype=tf.float32, shape=[], initializer=init)
rcomputed = []
auc = []
sess = tf.Session()
sess.run(tf.initialize_all_variables())
print sess.run(tflearn.get_training_mode()) #False
tflearn.is_training(True, session=sess)
print sess.run(tflearn.get_training_mode()) #now True
X = data_train
D = X.shape[1]
nu = 0.04
# temp = np.random.normal(0, 1, K + K*D + 1)[-1]
def highway(incoming,
n_units,
activation='linear',
transform_dropout=None,
weights_init='truncated_normal',
bias_init='zeros',
regularizer=None,
weight_decay=0.001,
trainable=True,
restore=True,
reuse=False,
scope=None,
name="FullyConnectedHighway"):
""" Fully Connected Highway.
A fully connected highway network layer, with some inspiration from
[https://github.com/fomorians/highway-fcn](https://github.com/fomorians/highway-fcn).
Input:
(2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Output:
2D Tensor [samples, n_units].
Arguments:
incoming: `Tensor`. Incoming (2+)D Tensor.
n_units: `int`, number of units for this layer.
activation: `str` (name) or `function` (returning a `Tensor`).
Activation applied to this layer (see tflearn.activations).
Default: 'linear'.
transform_dropout: `float`: Keep probability on the highway transform gate.
weights_init: `str` (name) or `Tensor`. Weights initialization.
(see tflearn.initializations) Default: 'truncated_normal'.
bias_init: `str` (name) or `Tensor`. Bias initialization.
(see tflearn.initializations) Default: 'zeros'.
regularizer: `str` (name) or `Tensor`. Add a regularizer to this
layer weights (see tflearn.regularizers). Default: None.
weight_decay: `float`. Regularizer decay parameter. Default: 0.001.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model
reuse: `bool`. If True and 'scope' is provided, this layer variables
will be reused (shared).
scope: `str`. Define this layer scope (optional). A scope can be
used to share variables between layers. Note that scope will
override name.
name: A name for this layer (optional). Default: 'FullyConnectedHighway'.
Attributes:
scope: `Scope`. This layer scope.
W: `Tensor`. Variable representing units weights.
W_t: `Tensor`. Variable representing units weights for transform gate.
b: `Tensor`. Variable representing biases.
b_t: `Tensor`. Variable representing biases for transform gate.
Links:
[https://arxiv.org/abs/1505.00387](https://arxiv.org/abs/1505.00387)
"""
input_shape = utils.get_incoming_shape(incoming)
assert len(input_shape) > 1, "Incoming Tensor shape must be at least 2-D"
n_inputs = int(np.prod(input_shape[1:]))
# Build variables and inference.
with tf.variable_scope(scope,
default_name=name,
values=[incoming],
reuse=reuse) as scope:
name = scope.name
W_init = weights_init
if isinstance(weights_init, str):
W_init = initializations.get(weights_init)()
W_regul = None
if regularizer:
W_regul = lambda x: losses.get(regularizer)(x, weight_decay)
W = va.variable('W',
shape=[n_inputs, n_units],
regularizer=W_regul,
initializer=W_init,
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, W)
if isinstance(bias_init, str):
bias_init = initializations.get(bias_init)()
b = va.variable('b',
shape=[n_units],
initializer=bias_init,
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b)
# Weight and bias for the transform gate
W_T = va.variable('W_T',
shape=[n_inputs, n_units],
regularizer=None,
initializer=W_init,
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, W_T)
b_T = va.variable('b_T',
shape=[n_units],
initializer=tf.constant_initializer(-1),
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b_T)
# If input is not 2d, flatten it.
if len(input_shape) > 2:
incoming = tf.reshape(incoming, [-1, n_inputs])
if isinstance(activation, str):
activation = activations.get(activation)
elif hasattr(activation, '__call__'):
activation = activation
else:
raise ValueError("Invalid Activation.")
H = activation(tf.matmul(incoming, W) + b)
T = tf.sigmoid(tf.matmul(incoming, W_T) + b_T)
if transform_dropout:
T = dropout(T, transform_dropout)
C = tf.subtract(1.0, T)
inference = tf.add(tf.multiply(H, T), tf.multiply(incoming, C))
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, inference)
# Add attributes to Tensor to easy access weights.
inference.scope = scope
inference.W = W
inference.W_t = W_T
inference.b = b
inference.b_t = b_T
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def fully_connected(incoming,
n_units,
activation='linear',
bias=True,
weights_init='truncated_normal',
bias_init='zeros',
regularizer=None,
weight_decay=0.001,
trainable=True,
restore=True,
reuse=False,
scope=None,
name="FullyConnected"):
""" Fully Connected.
A fully connected layer.
Input:
(2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Output:
2D Tensor [samples, n_units].
Arguments:
incoming: `Tensor`. Incoming (2+)D Tensor.
n_units: `int`, number of units for this layer.
activation: `str` (name) or `function` (returning a `Tensor`).
Activation applied to this layer (see tflearn.activations).
Default: 'linear'.
bias: `bool`. If True, a bias is used.
weights_init: `str` (name) or `Tensor`. Weights initialization.
(see tflearn.initializations) Default: 'truncated_normal'.
bias_init: `str` (name) or `Tensor`. Bias initialization.
(see tflearn.initializations) Default: 'zeros'.
regularizer: `str` (name) or `Tensor`. Add a regularizer to this
layer weights (see tflearn.regularizers). Default: None.
weight_decay: `float`. Regularizer decay parameter. Default: 0.001.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model.
reuse: `bool`. If True and 'scope' is provided, this layer variables
will be reused (shared).
scope: `str`. Define this layer scope (optional). A scope can be
used to share variables between layers. Note that scope will
override name.
name: A name for this layer (optional). Default: 'FullyConnected'.
Attributes:
scope: `Scope`. This layer scope.
W: `Tensor`. Variable representing units weights.
b: `Tensor`. Variable representing biases.
"""
input_shape = utils.get_incoming_shape(incoming)
assert len(input_shape) > 1, "Incoming Tensor shape must be at least 2-D"
n_inputs = int(np.prod(input_shape[1:]))
with tf.variable_scope(scope,
default_name=name,
values=[incoming],
reuse=reuse) as scope:
name = scope.name
W_init = weights_init
if isinstance(weights_init, str):
W_init = initializations.get(weights_init)()
W_regul = None
if regularizer:
W_regul = lambda x: losses.get(regularizer)(x, weight_decay)
W = va.variable('W',
shape=[n_inputs, n_units],
regularizer=W_regul,
initializer=W_init,
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, W)
b = None
if bias:
if isinstance(bias_init, str):
bias_init = initializations.get(bias_init)()
b = va.variable('b',
shape=[n_units],
initializer=bias_init,
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 2:
inference = tf.reshape(inference, [-1, n_inputs])
inference = tf.matmul(inference, W)
if b: inference = tf.nn.bias_add(inference, b)
if activation:
if isinstance(activation, str):
inference = activations.get(activation)(inference)
elif hasattr(activation, '__call__'):
inference = activation(inference)
else:
raise ValueError("Invalid Activation.")
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, inference)
# Add attributes to Tensor to easy access weights.
inference.scope = scope
inference.W = W
inference.b = b
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def PhraseLayer(incoming,
input_dim,
output_dim,
output_length,
activation='linear',
dropout_keepprob=0.5,
batchNorm=False,
name='PhraseLayer',
alpha=0.5,
scope=None):
'''
incoming: [batch_size, sen_length, input_dim]
return: [batch_size, sen_length, output_length, output_dim[0]],
[batch_size, sen_length, output_length, output_dim[1]]
'''
with tf.variable_scope(scope, default_name=name,
values=[incoming]) as scope:
name = scope.name
P = va.variable('P',
shape=[input_dim, output_dim[0]],
initializer=initializations.get('truncated_normal')())
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, P)
P_p = va.variable(
'P_p',
shape=[input_dim, output_dim[1]],
initializer=initializations.get('truncated_normal')())
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, P_p)
Q = va.variable('Q',
shape=[input_dim, output_dim[0]],
initializer=initializations.get('truncated_normal')())
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, Q)
Q_p = va.variable(
'Q_p',
shape=[input_dim, output_dim[1]],
initializer=initializations.get('truncated_normal')())
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, Q_p)
R = va.variable('R',
shape=[input_dim, output_dim[0]],
initializer=initializations.get('truncated_normal')())
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, R)
R_p = va.variable(
'R_p',
shape=[input_dim, output_dim[1]],
initializer=initializations.get('truncated_normal')())
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, R_p)
O = va.variable('O',
shape=[output_dim[0], output_dim[0]],
initializer=initializations.get('truncated_normal')())
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, O)
O_p = va.variable('O_p',
shape=[output_dim[1], output_dim[1]],
initializer=tf.ones_initializer())
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, O_p)
b = va.variable('b',
shape=[1, 1, 1, output_dim[0]],
initializer=tf.ones_initializer())
b_p = va.variable('b_p',
shape=[1, 1, 1, output_dim[1]],
initializer=tf.ones_initializer())
if isinstance(activation, str):
activation = activations.get(activation)
elif hasattr(activation, '__call__'):
activation = activation
else:
raise ValueError("Invalid Activation.")
def calc(incoming, P, Q, R, O, output_dim):
batch_size = tf.shape(incoming)[0]
sent_length = incoming.shape[1].value
G1 = tf.zeros([batch_size, sent_length, output_dim])
G2 = tf.zeros([batch_size, sent_length, output_dim])
G3 = tf.zeros([batch_size, sent_length, output_dim])
r = []
for i in range(output_length):
'''if i == 0:
now = incoming
else:
now = tf.concat([tf.zeros([batch_size, i, input_dim]), incoming[:,0:-i, :]], axis = 1)
F2 = tf.einsum('aij,jk->aik', now, Q) * G1
F3 = tf.einsum('aij,jk->aik', now, R) * G2
if i == 0:
F1 = tf.einsum('aij,jk->aik', now, P)
G1 = G1 * alpha + F1
else:
G1 = G1 * alpha
G2 = G2 * alpha + F2
G3 = G3 * alpha + F3
r.append(tf.einsum('aij,jk->aik',G1+G2+G3, O))'''
F1 = tf.einsum('aij,jk->aik', incoming, P)
r.append(tf.einsum('aij,jk->aik', F1, O))
#return tf.stack(r, axis = 2)
return tf.reshape(r[0], [batch_size, sent_length, 1, output_dim])
batch_size = tf.shape(incoming)[0]
sent_length = incoming.shape[1].value
#out1 = tf.reshape(tf.einsum('aij,jk->aik', tf.einsum('aij,jk->aik', incoming, P), O), [batch_size, sent_length, 1, output_dim[0]]) + b
out1 = tf.reshape(tf.einsum('aij,jk->aik', incoming, P),
[batch_size, sent_length, 1, output_dim[0]]) + b
#out1 = calc(incoming, P, Q, R, O, output_dim[0]) + b
#out1 = activation(out1, name="activation")
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, out1)
if batchNorm:
pass
#out1 = tflearn.batch_normalization(out1, name="batchNormOut1")
#out1 = tflearn.dropout(out1, dropout_keepprob, name="dropOut1")
if output_dim[1] == 0:
out2 = None
else:
out2 = calc(tf.stop_gradient(incoming), P_p, Q_p, R_p, O_p,
output_dim[1]) + b_p
out2 = activation(out2, name="activation_p")
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, out2)
if batchNorm:
out2 = tflearn.batch_normalization(out2, name="batchNormOut2")
out2 = tflearn.dropout(out2, dropout_keepprob, name="dropOut2")
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, out1)
if output_dim[1] != 0:
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, out2)
return out1, out2
def _linear(args,
output_size,
bias,
bias_start=0.0,
weights_init=None,
trainable=True,
restore=True,
reuse=False,
scope=None):
"""Linear map: sum_i(args[i] * W[i]), where W[i] is a variable.
Arguments:
args: a 2D Tensor or a list of 2D, batch x n, Tensors.
output_size: int, second dimension of W[i].
bias: boolean, whether to add a bias term or not.
bias_start: starting value to initialize the bias; 0 by default.
scope: VariableScope for the created subgraph; defaults to "Linear".
Returns:
A 2D Tensor with shape [batch x output_size] equal to
sum_i(args[i] * W[i]), where W[i]s are newly created matrices.
Raises:
ValueError: if some of the arguments has unspecified or wrong shape.
"""
if args is None or (is_sequence(args) and not args):
raise ValueError("`args` must be specified")
if not is_sequence(args):
args = [args]
# Calculate the total size of arguments on dimension 1.
total_arg_size = 0
shapes = [a.get_shape().as_list() for a in args]
for shape in shapes:
if len(shape) != 2:
raise ValueError("Linear is expecting 2D arguments: %s" %
str(shapes))
if not shape[1]:
raise ValueError("Linear expects shape[1] of arguments: %s" %
str(shapes))
else:
total_arg_size += shape[1]
# Now the computation.
with tf.variable_scope(scope or "Linear", reuse=reuse):
matrix = va.variable("Matrix", [total_arg_size, output_size],
initializer=weights_init,
trainable=trainable,
restore=restore)
if len(args) == 1:
res = tf.matmul(args[0], matrix)
else:
res = tf.matmul(array_ops.concat(args, 1), matrix)
if not bias:
return res
bias_term = va.variable(
"Bias", [output_size],
initializer=tf.constant_initializer(bias_start),
trainable=trainable,
restore=restore)
return res + bias_term
def fully_connected(incoming,
n_units,
activation='linear',
bias=True,
weights_init='truncated_normal',
bias_init='zeros',
regularizer=None,
weight_decay=0.001,
trainable=True,
restore=True,
reuse=False,
scope=None,
name="FullyConnected"):
input_shape = utils.get_incoming_shape(incoming)
assert len(input_shape) > 1, "Incoming Tensor shape must be at least 2-D"
n_inputs = int(np.prod(input_shape[1:]))
with tf.variable_scope(scope,
default_name=name,
values=[incoming],
reuse=reuse) as scope:
name = scope.name
W_init = weights_init
if isinstance(weights_init, str):
W_init = initializations.get(weights_init)()
W_regul = None
if regularizer is not None:
W_regul = lambda x: losses.get(regularizer)(x, weight_decay)
W = vs.variable('W',
shape=[n_inputs, n_units],
regularizer=W_regul,
initializer=W_init,
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, W)
b = None
if bias:
if isinstance(bias_init, str):
bias_init = initializations.get(bias_init)()
b = vs.variable('b',
shape=[n_units],
initializer=bias_init,
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 2:
inference = tf.reshape(inference, [-1, n_inputs])
inference = tf.matmul(inference, W)
if b is not None: inference = tf.nn.bias_add(inference, b)
if activation:
if isinstance(activation, str):
inference = activations.get(activation)(inference)
elif hasattr(activation, '__call__'):
inference = activation(inference)
else:
raise ValueError("Invalid Activation.")
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, inference)
# Add attributes to Tensor to easy access weights.
inference.scope = scope
inference.W = W
inference.b = b
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def conv_2d(incoming,
nb_filter,
filter_size,
strides=1,
padding='same',
activation='linear',
bias=True,
weights_init='uniform_scaling',
bias_init='zeros',
regularizer=None,
weight_decay=0.001,
trainable=True,
restore=True,
reuse=False,
scope=None,
name="Conv2D"):
input_shape = utils.get_incoming_shape(incoming)
assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D"
filter_size = utils.autoformat_filter_conv2d(filter_size, input_shape[-1],
nb_filter)
strides = utils.autoformat_kernel_2d(strides)
padding = utils.autoformat_padding(padding)
with tf.variable_scope(scope,
default_name=name,
values=[incoming],
reuse=reuse) as scope:
name = scope.name
W_init = weights_init
if isinstance(weights_init, str):
W_init = initializations.get(weights_init)()
W_regul = None
if regularizer is not None:
W_regul = lambda x: losses.get(regularizer)(x, weight_decay)
W = vs.variable('W',
shape=filter_size,
regularizer=W_regul,
initializer=W_init,
trainable=trainable,
restore=restore)
# Track per layer variables
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, W)
b = None
if bias:
if isinstance(bias_init, str):
bias_init = initializations.get(bias_init)()
b = vs.variable('b',
shape=nb_filter,
initializer=bias_init,
trainable=trainable,
restore=restore)
# Track per layer variables
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b)
inference = tf.nn.conv2d(incoming, W, strides, padding)
if b is not None: inference = tf.nn.bias_add(inference, b)
if activation:
if isinstance(activation, str):
inference = activations.get(activation)(inference)
elif hasattr(activation, '__call__'):
inference = activation(inference)
else:
raise ValueError("Invalid Activation.")
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, inference)
# Add attributes to Tensor to easy access weights.
inference.scope = scope
inference.W = W
inference.b = b
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def highway(incoming, n_units, activation='linear', transform_dropout=None,
weights_init='truncated_normal', bias_init='zeros',
regularizer=None, weight_decay=0.001, trainable=True,
restore=True, reuse=False, scope=None,
name="FullyConnectedHighway"):
""" Fully Connected Highway.
A fully connected highway network layer, with some inspiration from
[https://github.com/fomorians/highway-fcn](https://github.com/fomorians/highway-fcn).
Input:
(2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Output:
2D Tensor [samples, n_units].
Arguments:
incoming: `Tensor`. Incoming (2+)D Tensor.
n_units: `int`, number of units for this layer.
activation: `str` (name) or `function` (returning a `Tensor`).
Activation applied to this layer (see tflearn.activations).
Default: 'linear'.
transform_dropout: `float`: Keep probability on the highway transform gate.
weights_init: `str` (name) or `Tensor`. Weights initialization.
(see tflearn.initializations) Default: 'truncated_normal'.
bias_init: `str` (name) or `Tensor`. Bias initialization.
(see tflearn.initializations) Default: 'zeros'.
regularizer: `str` (name) or `Tensor`. Add a regularizer to this
layer weights (see tflearn.regularizers). Default: None.
weight_decay: `float`. Regularizer decay parameter. Default: 0.001.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model
reuse: `bool`. If True and 'scope' is provided, this layer variables
will be reused (shared).
scope: `str`. Define this layer scope (optional). A scope can be
used to share variables between layers. Note that scope will
override name.
name: A name for this layer (optional). Default: 'FullyConnectedHighway'.
Attributes:
scope: `Scope`. This layer scope.
W: `Tensor`. Variable representing units weights.
W_t: `Tensor`. Variable representing units weights for transform gate.
b: `Tensor`. Variable representing biases.
b_t: `Tensor`. Variable representing biases for transform gate.
Links:
[https://arxiv.org/abs/1505.00387](https://arxiv.org/abs/1505.00387)
"""
input_shape = utils.get_incoming_shape(incoming)
assert len(input_shape) > 1, "Incoming Tensor shape must be at least 2-D"
n_inputs = int(np.prod(input_shape[1:]))
# Build variables and inference.
with tf.variable_op_scope([incoming], scope, name, reuse=reuse) as scope:
name = scope.name
W_init = weights_init
if isinstance(weights_init, str):
W_init = initializations.get(weights_init)()
W_regul = None
if regularizer:
W_regul = lambda x: losses.get(regularizer)(x, weight_decay)
W = va.variable('W', shape=[n_inputs, n_units], regularizer=W_regul,
initializer=W_init, trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, W)
if isinstance(bias_init, str):
bias_init = initializations.get(bias_init)()
b = va.variable('b', shape=[n_units], initializer=bias_init,
trainable=trainable, restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b)
# Weight and bias for the transform gate
W_T = va.variable('W_T', shape=[n_inputs, n_units],
regularizer=None, initializer=W_init,
trainable=trainable, restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, W_T)
b_T = va.variable('b_T', shape=[n_units],
initializer=tf.constant_initializer(-1),
trainable=trainable, restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b_T)
# If input is not 2d, flatten it.
if len(input_shape) > 2:
incoming = tf.reshape(incoming, [-1, n_inputs])
if isinstance(activation, str):
activation = activations.get(activation)
elif hasattr(activation, '__call__'):
activation = activation
else:
raise ValueError("Invalid Activation.")
H = activation(tf.matmul(incoming, W) + b)
T = tf.sigmoid(tf.matmul(incoming, W_T) + b_T)
if transform_dropout:
T = dropout(T, transform_dropout)
C = tf.sub(1.0, T)
inference = tf.add(tf.mul(H, T), tf.mul(incoming, C))
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, inference)
# Add attributes to Tensor to easy access weights.
inference.scope = scope
inference.W = W
inference.W_t = W_T
inference.b = b
inference.b_t = b_T
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def fully_connected(incoming, n_units, activation='linear', bias=True,
weights_init='truncated_normal', bias_init='zeros',
regularizer=None, weight_decay=0.001, trainable=True,
restore=True, reuse=False, scope=None,
name="FullyConnected"):
""" Fully Connected.
A fully connected layer.
Input:
(2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Output:
2D Tensor [samples, n_units].
Arguments:
incoming: `Tensor`. Incoming (2+)D Tensor.
n_units: `int`, number of units for this layer.
activation: `str` (name) or `function` (returning a `Tensor`).
Activation applied to this layer (see tflearn.activations).
Default: 'linear'.
bias: `bool`. If True, a bias is used.
weights_init: `str` (name) or `Tensor`. Weights initialization.
(see tflearn.initializations) Default: 'truncated_normal'.
bias_init: `str` (name) or `Tensor`. Bias initialization.
(see tflearn.initializations) Default: 'zeros'.
regularizer: `str` (name) or `Tensor`. Add a regularizer to this
layer weights (see tflearn.regularizers). Default: None.
weight_decay: `float`. Regularizer decay parameter. Default: 0.001.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model.
reuse: `bool`. If True and 'scope' is provided, this layer variables
will be reused (shared).
scope: `str`. Define this layer scope (optional). A scope can be
used to share variables between layers. Note that scope will
override name.
name: A name for this layer (optional). Default: 'FullyConnected'.
Attributes:
scope: `Scope`. This layer scope.
W: `Tensor`. Variable representing units weights.
b: `Tensor`. Variable representing biases.
"""
input_shape = utils.get_incoming_shape(incoming)
assert len(input_shape) > 1, "Incoming Tensor shape must be at least 2-D"
n_inputs = int(np.prod(input_shape[1:]))
# Build variables and inference.
with tf.variable_op_scope([incoming], scope, name, reuse=reuse) as scope:
name = scope.name
W_init = weights_init
if isinstance(weights_init, str):
W_init = initializations.get(weights_init)()
W_regul = None
if regularizer:
W_regul = lambda x: losses.get(regularizer)(x, weight_decay)
W = va.variable('W', shape=[n_inputs, n_units], regularizer=W_regul,
initializer=W_init, trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, W)
b = None
if bias:
if isinstance(bias, str):
bias_init = initializations.get(bias_init)()
b = va.variable('b', shape=[n_units], initializer=bias_init,
trainable=trainable, restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 2:
inference = tf.reshape(inference, [-1, n_inputs])
inference = tf.matmul(inference, W)
if b: inference = tf.nn.bias_add(inference, b)
if isinstance(activation, str):
inference = activations.get(activation)(inference)
elif hasattr(activation, '__call__'):
inference = activation(inference)
else:
raise ValueError("Invalid Activation.")
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, inference)
# Add attributes to Tensor to easy access weights.
inference.scope = scope
inference.W = W
inference.b = b
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def tflearn_OneClass_NN_linear(data_train, data_test, labels_train):
X = data_train
Y = labels_train
D = X.shape[1]
No_of_inputNodes = X.shape[1]
# Clear all the graph variables created in previous run and start fresh
tf.reset_default_graph()
# Define the network
input_layer = input_data(shape=[None,
No_of_inputNodes]) # input layer of size
np.random.seed(42)
theta0 = np.random.normal(0, 1, K + K * D + 1) * 0.0001
#theta0 = np.random.normal(0, 1, K + K*D + 1) # For linear
hidden_layer = fully_connected(
input_layer,
4,
bias=False,
activation='linear',
name="hiddenLayer_Weights",
weights_init="normal") # hidden layer of size 2
output_layer = fully_connected(
hidden_layer,
1,
bias=False,
activation='linear',
name="outputLayer_Weights",
weights_init="normal") # output layer of size 1
# Initialize rho
value = 0.01
init = tf.constant_initializer(value)
rho = va.variable(name='rho', dtype=tf.float32, shape=[], initializer=init)
rcomputed = []
auc = []
sess = tf.Session()
sess.run(tf.initialize_all_variables())
# print sess.run(tflearn.get_training_mode()) #False
tflearn.is_training(True, session=sess)
print sess.run(tflearn.get_training_mode()) #now True
temp = theta0[-1]
oneClassNN_Net = oneClassNN(output_layer,
v,
rho,
hidden_layer,
output_layer,
optimizer='sgd',
loss='OneClassNN_Loss',
learning_rate=1)
model = DNN(oneClassNN_Net, tensorboard_verbose=3)
model.set_weights(output_layer.W, theta0[0:K][:, np.newaxis])
model.set_weights(hidden_layer.W, np.reshape(theta0[K:K + K * D], (D, K)))
iterStep = 0
while (iterStep < 100):
print "Running Iteration :", iterStep
# Call the cost function
y_pred = model.predict(data_train) # Apply some ops
tflearn.is_training(False, session=sess)
y_pred_test = model.predict(data_test) # Apply some ops
tflearn.is_training(True, session=sess)
value = np.percentile(y_pred, v * 100)
tflearn.variables.set_value(rho, value, session=sess)
rStar = rho
model.fit(X, Y, n_epoch=2, show_metric=True, batch_size=100)
iterStep = iterStep + 1
rcomputed.append(rho)
temp = tflearn.variables.get_value(rho, session=sess)
# print "Rho",temp
# print "y_pred",y_pred
# print "y_predTest", y_pred_test
# g = lambda x: x
g = lambda x: 1 / (1 + tf.exp(-x))
def nnScore(X, w, V, g):
return tf.matmul(g((tf.matmul(X, w))), V)
# Format the datatype to suite the computation of nnscore
X = X.astype(np.float32)
X_test = data_test
X_test = X_test.astype(np.float32)
# assign the learnt weights
# wStar = hidden_layer.W
# VStar = output_layer.W
# Get weights values of fc2
wStar = model.get_weights(hidden_layer.W)
VStar = model.get_weights(output_layer.W)
# print "Hideen",wStar
# print VStar
train = nnScore(X, wStar, VStar, g)
test = nnScore(X_test, wStar, VStar, g)
# Access the value inside the train and test for plotting
# Create a new session and run the example
# sess = tf.Session()
# sess.run(tf.initialize_all_variables())
arrayTrain = train.eval(session=sess)
arrayTest = test.eval(session=sess)
# print "Train Array:",arrayTrain
# print "Test Array:",arrayTest
# plt.hist(arrayTrain-temp, bins = 25,label='Normal');
# plt.hist(arrayTest-temp, bins = 25, label='Anomalies');
# plt.legend(loc='upper right')
# plt.title('r = %1.6f- Sigmoid Activation ' % temp)
# plt.show()
pos_decisionScore = arrayTrain - temp
neg_decisionScore = arrayTest - temp
return [pos_decisionScore, neg_decisionScore]
def single_unit(incoming,
activation='linear',
bias=True,
trainable=True,
restore=True,
reuse=False,
scope=None,
name="Linear"):
""" Single Unit.
A single unit (Linear) Layer.
Input:
1-D Tensor [samples]. If not 2D, input will be flatten.
Output:
1-D Tensor [samples].
Arguments:
incoming: `Tensor`. Incoming Tensor.
activation: `str` (name) or `function`. Activation applied to this
layer (see tflearn.activations). Default: 'linear'.
bias: `bool`. If True, a bias is used.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model.
reuse: `bool`. If True and 'scope' is provided, this layer variables
will be reused (shared).
scope: `str`. Define this layer scope (optional). A scope can be
used to share variables between layers. Note that scope will
override name.
name: A name for this layer (optional). Default: 'Linear'.
Attributes:
W: `Tensor`. Variable representing weight.
b: `Tensor`. Variable representing bias.
"""
input_shape = utils.get_incoming_shape(incoming)
n_inputs = int(np.prod(input_shape[1:]))
# Build variables and inference.
with tf.variable_scope(scope,
default_name=name,
values=[incoming],
reuse=reuse) as scope:
name = scope.name
W = va.variable('W',
shape=[n_inputs],
initializer=tf.constant_initializer(np.random.randn()),
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, W)
b = None
if bias:
b = va.variable('b',
shape=[n_inputs],
initializer=tf.constant_initializer(
np.random.randn()),
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 1:
inference = tf.reshape(inference, [-1])
inference = tf.multiply(inference, W)
if b: inference = tf.add(inference, b)
if isinstance(activation, str):
inference = activations.get(activation)(inference)
elif hasattr(activation, '__call__'):
inference = activation(inference)
else:
raise ValueError("Invalid Activation.")
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, inference)
# Add attributes to Tensor to easy access weights.
inference.scope = scope
inference.W = W
inference.b = b
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def single_unit(incoming, activation='linear', bias=True, trainable=True,
restore=True, name="Linear"):
""" Single Unit.
A single unit (Linear) Layer.
Input:
1-D Tensor [samples]. If not 2D, input will be flatten.
Output:
1-D Tensor [samples].
Arguments:
incoming: `Tensor`. Incoming Tensor.
activation: `str` (name) or `Tensor`. Activation applied to this layer.
(see tflearn.activations). Default: 'linear'.
bias: `bool`. If True, a bias is used.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model.
name: A name for this layer (optional). Default: 'Dense'.
Attributes:
W: `Tensor`. Variable representing weight.
b: `Tensor`. Variable representing bias.
"""
input_shape = utils.get_incoming_shape(incoming)
n_inputs = int(np.prod(input_shape[1:]))
# Build variables and inference.
with tf.name_scope(name) as scope:
W = va.variable(scope + 'W', shape=[n_inputs],
initializer=tf.constant_initializer(np.random.randn()),
trainable=trainable, restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope, W)
b = None
if bias:
b = va.variable(scope + 'b', shape=[n_inputs],
initializer=tf.constant_initializer(np.random.randn()),
trainable=trainable, restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope, b)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 1:
inference = tf.reshape(inference, [-1])
inference = tf.mul(inference, W)
if b: inference = tf.add(inference, b)
inference = activations.get(activation)(inference)
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, inference)
# Add attributes to Tensor to easy access weights.
inference.scope = scope
inference.W = W
inference.b = b
return inference
def conv_1d_tranpose(layer,
nb_filter,
filter_size,
strides,
padding='same',
bias=True,
scope=None,
reuse=False,
bias_init='zeros',
trainable=True,
restore=True,
regularizer=None,
weight_decay=0.001,
weights_init='uniform_scaling',
name="deconv_1d"):
'''
layer: A 3-D `Tensor` of type `float` and shape `[batch, in_width, in_channels]` .
SEE: https://www.tensorflow.org/api_docs/python/tf/nn/conv2d_backprop_input
SEE2: https://github.com/tensorflow/tensorflow/pull/13105/commits/2ca9b908d1978a94855349309fd16a67cfd98659
TODO: ADD weight-decay/regularizer
'''
input_shape = utils.get_incoming_shape(layer)
_, in_width, in_channels = input_shape
batch_size = tf.shape(layer)[0]
filter_size = [filter_size, nb_filter, in_channels]
output_shape = [batch_size, strides * in_width, nb_filter
] # this trick I think work only for strict up-sampling
output_shape_ = ops.convert_to_tensor(output_shape, name="output_shape")
strides = [1, 1, strides, 1]
spatial_start_dim = 1
padding = utils.autoformat_padding(padding)
with tf.variable_scope(scope,
default_name=name,
values=[layer],
reuse=reuse) as scope:
name = scope.name
W_init = weights_init
if isinstance(weights_init, str):
W_init = initializations.get(weights_init)()
elif type(W_init) in [tf.Tensor, np.ndarray, list]:
filter_size = None
W_regul = None
if regularizer is not None:
W_regul = lambda x: tflearn.losses.get(regularizer)(x, weight_decay
)
W = vs.variable('W',
shape=filter_size,
regularizer=W_regul,
initializer=W_init,
trainable=trainable,
restore=restore)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, W)
# expand dims to make it compatible with conv2d
W = tf.expand_dims(W, 0)
layer = tf.expand_dims(layer, spatial_start_dim)
output_shape_ = array_ops.concat(
[output_shape_[:1], [1], output_shape_[1:]], axis=0)
result = gen_nn_ops.conv2d_backprop_input(input_sizes=output_shape_,
filter=W,
out_backprop=layer,
strides=strides,
padding=padding,
name=name)
result = array_ops.squeeze(result, [spatial_start_dim])
result = tf.reshape(result, shape=output_shape)
if bias:
b_shape = [nb_filter]
bias_init = initializations.get(bias_init)()
b = vs.variable('b',
shape=b_shape,
initializer=bias_init,
trainable=trainable,
restore=restore)
# Track per layer variables
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b)
result = tf.nn.bias_add(result, b)
result.b = b
result.scope = scope
result.W = W
return result
def conv_2d_BN(incoming, nb_filter, filter_size, strides=1, padding='same',
activation='linear', bias=True, weights_init='xavier',
bias_init='zeros', regularizer=None, weight_decay=0.001,
trainable=True, restore=True, reuse=False, scope=None,
name="Conv2D", batch_norm=False):
""" Convolution 2D.
Input:
4-D Tensor [batch, height, width, in_channels].
Output:
4-D Tensor [batch, new height, new width, nb_filter].
Arguments:
incoming: `Tensor`. Incoming 4-D Tensor.
nb_filter: `int`. The number of convolutional filters.
filter_size: `int` or `list of int`. Size of filters.
strides: 'int` or list of `int`. Strides of conv operation.
Default: [1 1 1 1].
padding: `str` from `"same", "valid"`. Padding algo to use.
Default: 'same'.
activation: `str` (name) or `function` (returning a `Tensor`).
Activation applied to this layer (see tflearn.activations).
Default: 'linear'.
bias: `bool`. If True, a bias is used.
weights_init: `str` (name) or `Tensor`. Weights initialization.
(see tflearn.initializations) Default: 'truncated_normal'.
bias_init: `str` (name) or `Tensor`. Bias initialization.
(see tflearn.initializations) Default: 'zeros'.
regularizer: `str` (name) or `Tensor`. Add a regularizer to this
layer weights (see tflearn.regularizers). Default: None.
weight_decay: `float`. Regularizer decay parameter. Default: 0.001.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model.
reuse: `bool`. If True and 'scope' is provided, this layer variables
will be reused (shared).
scope: `str`. Define this layer scope (optional). A scope can be
used to share variables between layers. Note that scope will
override name.
name: A name for this layer (optional). Default: 'Conv2D'.
batch_norm: If true, add batch normalization with default TFLearn
parameters before the activation layer
Attributes:
scope: `Scope`. This layer scope.
W: `Variable`. Variable representing filter weights.
b: `Variable`. Variable representing biases.
"""
input_shape = utils.get_incoming_shape(incoming)
assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D"
filter_size = utils.autoformat_filter_conv2d(filter_size,
input_shape[-1],
nb_filter)
strides = utils.autoformat_kernel_2d(strides)
padding = utils.autoformat_padding(padding)
# Variable Scope fix for older TF
try:
vscope = tf.variable_scope(scope, default_name=name, values=[incoming],
reuse=reuse)
except Exception:
vscope = tf.variable_op_scope([incoming], scope, name, reuse=reuse)
with vscope as scope:
name = scope.name
W_init = weights_init
if isinstance(weights_init, str):
W_init = initializations.get(weights_init)()
W_regul = None
if regularizer:
W_regul = lambda x: losses.get(regularizer)(x, weight_decay)
W = vs.variable('W', shape=filter_size, regularizer=W_regul,
initializer=W_init, trainable=trainable,
restore=restore)
# Track per layer variables
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, W)
b = None
if bias:
if isinstance(bias_init, str):
bias_init = initializations.get(bias_init)()
b = vs.variable('b', shape=nb_filter, initializer=bias_init,
trainable=trainable, restore=restore)
# Track per layer variables
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b)
inference = tf.nn.conv2d(incoming, W, strides, padding)
if b: inference = tf.nn.bias_add(inference, b)
if batch_norm:
inference = batch_normalization(inference)
if isinstance(activation, str):
if activation == 'softmax':
shapes = inference.get_shape()
inference = activations.get(activation)(inference)
elif hasattr(activation, '__call__'):
inference = activation(inference)
else:
raise ValueError("Invalid Activation.")
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, inference)
# Add attributes to Tensor to easy access weights.
inference.scope = scope
inference.W = W
inference.b = b
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
