def step(h, x):
# simple rnn
### Replace tensor[:, i, :] with x. bryan ###
y = ln(
tf.matmul(tensor[:, i, :], w) + tf.matmul(h, u) +
(b if opt.bias else 0))
return y
def _linear(self, arys):
scope = tf.get_variable_scope()
with tf.variable_scope(scope, reuse=True):
w_i2h = tf.get_variable("w_i2h")
w_h2h = tf.get_variable("w_h2h")
w_b = tf.get_variable("w_b") if self._bias == True else 0
i2h = tf.matmul(arys[0], w_i2h)
h2h = tf.matmul(arys[1], w_h2h)
out = i2h + h2h + w_b
return out
def step(h, c, x):
# forget gate
f = tf.sigmoid(
ln(
tf.matmul(x, w_f) + tf.matmul(h, u_f) +
(b_f if opt.bias else 0)))
# input gate
i = tf.sigmoid(
ln(
tf.matmul(x, w_i) + tf.matmul(h, u_i) +
(b_i if opt.bias else 0)))
# new cell value
cc = tf.tanh(
ln(
tf.matmul(x, w_c) + tf.matmul(h, u_c) +
(b_c if opt.bias else 0)))
# out gate
o = tf.sigmoid(
ln(
tf.matmul(x, w_o) + tf.matmul(h, u_o) +
(b_o if opt.bias else 0)))
# cell update
cell = f * c + i * cc
# final output
y = o * tf.tanh(cell)
return y, cell
def sg_dense(tensor, opt):
r"""Applies a full connection.
Args:
tensor: A 2-D tensor (automatically passed by decorator).
opt:
in_dim: An `integer`. The size of input dimension.
dim: An `integer`. The size of output dimension.
bias: Boolean. If True, biases are added.
regularizer: A (Tensor -> Tensor or None) function; the result of applying it on a newly created variable
will be added to the collection tf.GraphKeys.REGULARIZATION_LOSSES and can be used for regularization
summary: If True, summaries are added. The default is True.
Returns:
A `Tensor` with the same type as `tensor`.
"""
# parameter initialize
w = tf.sg_initializer.he_uniform('W', (opt.in_dim, opt.dim),
regularizer=opt.regularizer, summary=opt.summary)
b = tf.sg_initializer.constant('b', opt.dim, summary=opt.summary) if opt.bias else 0
# apply transform
out = tf.matmul(tensor, w) + b
return out
def linear(input_, output_size, scope=None):
'''
Linear map: output[k] = sum_i(Matrix[k, i] * args[i] ) + Bias[k]
Args:
args: a tensor or a list of 2D, batch x n, Tensors.
output_size: int, second dimension of W[i].
scope: VariableScope for the created subgraph; defaults to "Linear".
Returns:
A 2D Tensor with shape [batch x output_size] equal to
sum_i(args[i] * W[i]), where W[i]s are newly created matrices.
Raises:
ValueError: if some of the arguments has unspecified or wrong shape.
'''
shape = input_.get_shape().as_list()
if len(shape) != 2:
raise ValueError("Linear is expecting 2D arguments: %s" % str(shape))
if not shape[1]:
raise ValueError("Linear expects shape[1] of arguments: %s" %
str(shape))
input_size = shape[1]
# Now the computation.
with tf.variable_scope(scope or "SimpleLinear"):
matrix = tf.get_variable("Matrix", [output_size, input_size],
dtype=input_.dtype)
bias_term = tf.get_variable("Bias", [output_size], dtype=input_.dtype)
return tf.matmul(input_, tf.transpose(matrix)) + bias_term
def chamfer_loss(A,B):
r=tf.reduce_sum(A*A,2)
r=tf.reshape(r,[int(r.shape[0]),int(r.shape[1]),1])
r2=tf.reduce_sum(B*B,2)
r2=tf.reshape(r2,[int(r.shape[0]),int(r.shape[1]),1])
t=(r-2*tf.matmul(A, tf.transpose(B,perm=[0, 2, 1])) + tf.transpose(r2,perm=[0, 2, 1]))
return tf.reduce_mean((tf.reduce_min(t, axis=1)+tf.reduce_min(t,axis=2))/2.0)
def sg_dense(tensor, opt):
# parameter initialize
w = init.he_uniform('W', (opt.in_dim, opt.dim))
if opt.bias:
b = init.constant('b', opt.dim)
# apply transform
out = tf.matmul(tensor, w) + (b if opt.bias else 0)
return out
def step(h, x):
# layer normalization
def ln(xx, opt):
if opt.ln:
# calc layer mean, variance for final axis
mean, variance = tf.nn.moments(xx, axes=[len(xx.get_shape()) - 1])
# apply layer normalization ( explicit broadcasting needed )
broadcast_shape = [-1] + [1] * (len(xx.get_shape()) - 1)
xx = (xx - tf.reshape(mean, broadcast_shape)) \
/ tf.reshape(tf.sqrt(variance + tf.sg_eps), broadcast_shape)
# apply parameter
return gamma * xx + beta
# apply transform
y = ln(tf.matmul(x, w) + tf.matmul(h, u) + (b if opt.bias else 0), opt)
return y
def step(h, x):
# layer normalization
def ln(xx, opt):
if opt.ln:
# calc layer mean, variance for final axis
mean, variance = tf.nn.moments(xx, axes=[len(xx.get_shape()) - 1])
# apply layer normalization ( explicit broadcasting needed )
broadcast_shape = [-1] + [1] * (len(xx.get_shape()) - 1)
xx = (xx - tf.reshape(mean, broadcast_shape)) \
/ tf.reshape(tf.sqrt(variance + tf.sg_eps), broadcast_shape)
# apply parameter
return gamma * xx + beta
# update gate
z = tf.sigmoid(ln(tf.matmul(x, w_z) + tf.matmul(h, u_z) + (b_z if opt.bias else 0), opt))
# reset gate
r = tf.sigmoid(ln(tf.matmul(x, w_r) + tf.matmul(h, u_r) + (b_r if opt.bias else 0), opt))
# h_hat
hh = tf.sigmoid(ln(tf.matmul(x, w_h) + tf.matmul(r*h, u_h) + (b_h if opt.bias else 0), opt))
# final output
y = (1. - z) * h + z * hh
return y
def step(hh, x):
# update gate
z = tf.sigmoid(ln(tf.matmul(x, w_z) + tf.matmul(hh, u_z) + (b_z if opt.bias else 0)))
# reset gate
r = tf.sigmoid(ln(tf.matmul(x, w_r) + tf.matmul(hh, u_r) + (b_r if opt.bias else 0)))
# h_hat
h_hat = tf.tanh(ln(tf.matmul(x, w_h) + tf.matmul(r * hh, u_h) + (b_h if opt.bias else 0)))
# final output
y = (1. - z) * h_hat + z * hh
return y
def sg_dense(tensor, opt):
r"""Applies a full connection.
Args:
tensor: A 2-D `Tensor`.
in_dim: An `integer`. The size of input dimension.
dim: An `integer`. The size of output dimension.
bias: Boolean. If True, biases are added.
Returns:
A `Tensor` with the same type as `tensor`.
"""
# parameter initialize
w = init.he_uniform('W', (opt.in_dim, opt.dim))
if opt.bias:
b = init.constant('b', opt.dim)
# apply transform
out = tf.matmul(tensor, w) + (b if opt.bias else 0)
return out
def sg_dense(tensor, opt):
r"""Applies a full connection.
Args:
tensor: A 2-D tensor (automatically passed by decorator).
opt:
in_dim: An `integer`. The size of input dimension.
dim: An `integer`. The size of output dimension.
bias: Boolean. If True, biases are added.
Returns:
A `Tensor` with the same type as `tensor`.
"""
# parameter initialize
w = tf.sg_initializer.he_uniform('W', (opt.in_dim, opt.dim))
b = tf.sg_initializer.constant('b', opt.dim) if opt.bias else 0
# apply transform
out = tf.matmul(tensor, w) + b
return out
def get_gram_mat(tensor):
'''
Arg:
tensor: 4-D tensor. The first dimension must be 1.
Returns:
gram matrix. Read `https://en.wikipedia.org/wiki/Gramian_matrix` for details.
512 by 512.
'''
assert tensor.get_shape(
).ndims == 4, "The tensor must be 4 dimensions."
dim0, dim1, dim2, dim3 = tensor.get_shape().as_list()
tensor = tensor.sg_reshape(shape=[dim0 * dim1 * dim2,
dim3]) #(1*7*7, 512)
# normalization: Why? Because the original value of gram mat. would be too huge.
mean, variance = tf.nn.moments(tensor, [0, 1])
tensor = (tensor - mean) / tf.sqrt(variance + tf.sg_eps)
tensor_t = tensor.sg_transpose(perm=[1, 0]) #(512, 1*7*7)
gram_mat = tf.matmul(tensor_t, tensor) # (512, 512)
return gram_mat
def step(hh, x):
# simple rnn
y = ln(tf.matmul(x, w) + tf.matmul(hh, u) + (b if opt.bias else 0))
return y
batch_s, TF_max_timesteps = shape[0], shape[1]
with tf.name_scope('outputs'):
outputs = tf.reshape(outputs, [-1, num_hidden])
with tf.name_scope('weights'):
W = tf.Variable(tf.truncated_normal([num_hidden, num_classes],
stddev=0.1),
name='weights')
with tf.name_scope('biases'):
b = tf.get_variable("b",
initializer=tf.constant(0.,
shape=[num_classes]))
with tf.name_scope('logits'):
logits = tf.matmul(outputs, W) + b
logits = tf.reshape(logits, [batch_s, -1, num_classes])
logits = tf.transpose(logits, (1, 0, 2), name="out/logits")
with tf.name_scope('loss'):
loss = tf.nn.ctc_loss(targets,
logits,
seq_len,
ctc_merge_repeated=True,
preprocess_collapse_repeated=True)
with tf.name_scope('cost'):
cost = tf.reduce_mean(loss)
tf.summary.scalar("cost", cost)
with tf.name_scope('optimizer'):
optimizer = tf.train.RMSPropOptimizer(learning_rate,
decay=decay,
momentum=momentum,
act='leaky_relu',
bn=False)
d_p4 = ops.upconv_and_scale(d_p3,
dim=1,
size=size,
stride=stride,
act='linear',
bn=False)
disc = d_p4
#
# pull-away term ( PT ) regularizer
#
sample = gen.sg_flatten()
nom = tf.matmul(sample, tf.transpose(sample, perm=[1, 0]))
denom = tf.reduce_sum(tf.square(sample), reduction_indices=[1], keep_dims=True)
pt = tf.square(nom / denom)
pt -= tf.diag(tf.diag_part(pt))
pt = tf.reduce_sum(pt) / (batch_size * (batch_size - 1))
#
# loss & train ops
#
# mean squared errors
mse = tf.reduce_mean(tf.square(disc - xx), reduction_indices=[1, 2, 3])
mse_real, mse_fake = mse[:batch_size], mse[batch_size:]
loss_disc = mse_real + tf.maximum(margin - mse_fake, 0) # discriminator loss
loss_gen = mse_fake + pt * pt_weight # generator loss + PT regularizer
def step(h, x):
# simple rnn
y = ln(
tf.matmul(tensor[:, i, :], w) + tf.matmul(h, u) +
(b if opt.bias else 0))
return y
def t_get_indices(batchsize):
index = np.arange(batchsize)
np.random.shuffle(index)
return index
## Training Loop
sd = 1 / np.sqrt(num_features)
with tf.name_scope('input'):
X = tf.placeholder(tf.float32, [None, num_features], name="x_inp")
Y = tf.placeholder(tf.float32, [None, num_classes], name="y_inp")
W_1 = tf.Variable(
tf.random_normal([num_features, n_hidden_units_one], mean=0, stddev=sd))
b_1 = tf.Variable(tf.random_normal([n_hidden_units_one], mean=0, stddev=sd))
h_1 = tf.nn.tanh(tf.matmul(X, W_1) + b_1)
W_2 = tf.Variable(
tf.random_normal([n_hidden_units_one, n_hidden_units_two],
mean=0,
stddev=sd))
b_2 = tf.Variable(tf.random_normal([n_hidden_units_two], mean=0, stddev=sd))
h_2 = tf.nn.tanh(tf.matmul(h_1, W_2) + b_2)
W_3 = tf.Variable(
tf.random_normal([n_hidden_units_two, n_hidden_units_three],
mean=0,
stddev=sd))
b_3 = tf.Variable(tf.random_normal([n_hidden_units_three], mean=0, stddev=sd))
h_3 = tf.nn.sigmoid(tf.matmul(h_2, W_3) + b_3)
