def conv2d(inputs,
out_dim, k_size, strides,
padding="SAME",
w_init=None,
use_bias=True,
spectral_normed=True,
name="conv2d",
):
with tf.variable_scope(name):
w = tf.get_variable("w",
shape=[k_size, k_size, inputs.get_shape()[-1], out_dim],
dtype=tf.float32,
initializer=w_init
)
if spectral_normed:
w = spectral_normed_weight(w)
conv = tf.nn.conv2d(inputs, w, strides=[1, strides, strides, 1], padding=padding.upper())
if use_bias:
biases = tf.get_variable("b", [out_dim], initializer=tf.constant_initializer(0.0))
conv = tf.nn.bias_add(conv, biases, name="conv_add_b")
return conv
def conv2d(input_, output_dim,
k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02,
sn_op=None,
name="conv2d"):
# Code from:
# https://github.com/carpedm20/DCGAN-tensorflow
with tf.variable_scope(name):
w = tf.get_variable(
'w',
[k_h, k_w, input_.get_shape()[-1], output_dim],
initializer=tf.truncated_normal_initializer(stddev=stddev))
if sn_op is not None:
w = spectral_normed_weight(w, update_collection=sn_op)
conv = tf.nn.conv2d(
input_,
w,
strides=[1, d_h, d_w, 1],
padding='SAME')
biases = tf.get_variable(
'biases', [output_dim], initializer=tf.constant_initializer(0.0))
conv = tf.reshape(
tf.nn.bias_add(conv, biases),
[-1] + conv.get_shape().as_list()[1:])
return conv
def linear(inputs,
out_dim,
w_init=None,
activation=None,
use_bias=True, bias_start=0.0,
spectral_normed=False,
name="linear",
):
with tf.variable_scope(name):
w = tf.get_variable("w",
shape=[ inputs.get_shape()[-1], out_dim ],
dtype=tf.float32,
initializer=w_init
)
if spectral_normed:
w = spectral_normed_weight(w)
mul = tf.matmul(inputs, w, name='linear_mul')
if use_bias:
bias = tf.get_variable("b", [out_dim], initializer=tf.constant_initializer(bias_start))
mul = tf.nn.bias_add(mul, bias, name="mul_add_b")
if not (activation is None):
mul = activation(mul)
return mul
def conv2d(input_,
output_dim,
k_h=5,
k_w=5,
d_h=2,
d_w=2,
stddev=0.02,
spectral_norm=False,
name="conv2d"):
with tf.variable_scope(name):
w = tf.get_variable(
'w', [k_h, k_w, input_.get_shape()[-1], output_dim],
initializer=tf.truncated_normal_initializer(stddev=stddev))
if spectral_norm:
w = sn.spectral_normed_weight(w, update_collection=None)
conv = tf.nn.conv2d(input_,
w,
strides=[1, d_h, d_w, 1],
padding='SAME')
biases = tf.get_variable('biases', [output_dim],
initializer=tf.constant_initializer(0.0))
conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape())
return conv
def embed_labels(inputs,
num_classes,
output_dim,
sn,
weight_decay_rate=1e-5,
reuse=None,
scope=None):
# TODO move regularizer definitions to model
weights_regularizer = ly.l2_regularizer(weight_decay_rate)
with tf.variable_scope(scope, 'embedding', [inputs], reuse=reuse) as sc:
inputs = tf.convert_to_tensor(inputs)
weights = tf.get_variable(
name="weights",
shape=(num_classes, output_dim),
initializer=init_ops.random_normal_initializer)
# Spectral Normalization
if sn:
weights = spectral_normed_weight(
weights,
num_iters=1,
update_collection=Config.SPECTRAL_NORM_UPDATE_OPS)
embed_out = tf.nn.embedding_lookup(weights, inputs)
return embed_out
def sndilated_conv_layer(x, filter_shape, dilation, stddev=0.02, update_collection=None):
filters = tf.get_variable(
name='weight',
shape=filter_shape,
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev=stddev),
trainable=True)
return tf.nn.atrous_conv2d(x, sn.spectral_normed_weight(filters, update_collection=update_collection), dilation, padding='SAME')
def fully_connected(inputs,
num_outputs,
sn,
activation_fn=None,
normalizer_fn=None,
normalizer_params=None,
weights_initializer=ly.xavier_initializer(),
weight_decay_rate=1e-6,
biases_initializer=init_ops.zeros_initializer(),
biases_regularizer=None,
reuse=None,
scope=None):
# TODO move regularizer definitions to model
weights_regularizer = ly.l2_regularizer(weight_decay_rate)
input_dim = inputs.get_shape().as_list()[1]
with tf.variable_scope(scope, 'fully_connected', [inputs],
reuse=reuse) as sc:
inputs = tf.convert_to_tensor(inputs)
weights = tf.get_variable(name="weights",
shape=(input_dim, num_outputs),
initializer=weights_initializer,
regularizer=weights_regularizer,
trainable=True,
dtype=inputs.dtype.base_dtype)
# Spectral Normalization
if sn:
weights = spectral_normed_weight(
weights,
num_iters=1,
update_collection=Config.SPECTRAL_NORM_UPDATE_OPS)
linear_out = tf.matmul(inputs, weights)
if biases_initializer is not None:
biases = tf.get_variable(name="biases",
shape=(num_outputs, ),
initializer=biases_initializer,
regularizer=biases_regularizer,
trainable=True,
dtype=inputs.dtype.base_dtype)
linear_out = tf.nn.bias_add(linear_out, biases)
# Apply normalizer function / layer.
if normalizer_fn is not None:
normalizer_params = normalizer_params or {}
linear_out = normalizer_fn(linear_out,
activation_fn=None,
**normalizer_params)
if activation_fn is not None:
linear_out = activation_fn(linear_out)
return linear_out
def linear(input, output_dim, scope='linear', stddev=0.01,use_SN=False, update_collection=None):
norm = tf.random_normal_initializer(stddev=stddev)
const = tf.constant_initializer(0.0)
with tf.variable_scope(scope):
w = tf.get_variable('weights', [input.get_shape()[1], output_dim], initializer=norm)
b = tf.get_variable('biases', [output_dim], initializer=const)
if use_SN:
print('using SN')
return tf.matmul(input,spectral_normed_weight(w, update_collection=update_collection))+b
else:
return tf.matmul(input, w) + b
def linear(input_,
output_size,
name="Linear",
stddev=0.01,
scale=1.0,
with_learnable_sn_scale=False,
with_sn=False,
bias_start=0.0,
with_w=False,
update_collection=None,
with_singular_values=False):
shape = input_.get_shape().as_list()
with tf.variable_scope(name):
scope_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
tf.get_variable_scope().name)
has_summary = any([('Matrix' in v.op.name) for v in scope_vars])
matrix = tf.get_variable("Matrix", [shape[1], output_size], tf.float32,
tf.random_normal_initializer(stddev=stddev))
if with_sn:
s = tf.get_variable('s',
shape=[1],
initializer=tf.constant_initializer(scale),
trainable=with_learnable_sn_scale,
dtype=tf.float32)
matrix_bar, sigma = spectral_normed_weight(
matrix, update_collection=update_collection, with_sigma=True)
matrix_bar = s * matrix_bar
mul = tf.matmul(input_, matrix_bar)
else:
mul = tf.matmul(input_, matrix)
bias = tf.get_variable("bias", [output_size],
initializer=tf.constant_initializer(bias_start))
if not has_summary:
if with_sn:
variable_summaries({'b': bias, 's': s, 'sigma_w': sigma})
variable_summaries({'W': matrix},
with_singular_values=with_singular_values)
else:
variable_summaries({'b': bias})
variable_summaries({'W': matrix},
with_singular_values=with_singular_values)
if with_w:
return mul + bias, matrix, bias
else:
return mul + bias
def gen_snconv(x,
cnum,
ksize,
stride=1,
rate=1,
name='conv',
padding='SAME',
activation=tf.nn.relu,
training=True):
"""Define spectral normalization conv for discriminator.
Args:
x: Input.
cnum: Channel number.
ksize: Kernel size.
Stride: Convolution stride.
Rate: Rate for or dilated conv.
name: Name of layers.
padding: Default to SYMMETRIC.
activation: Activation function after convolution.
training: If current graph is for training or inference, used for bn.
Returns:
tf.Tensor: output
"""
assert padding in ['SYMMETRIC', 'SAME', 'REFELECT']
if padding == 'SYMMETRIC' or padding == 'REFELECT':
p = int(rate * (ksize - 1) / 2)
x = tf.pad(x, [[0, 0], [p, p], [p, p], [0, 0]], mode=padding)
padding = 'VALID'
fan_in = ksize * ksize * x.get_shape().as_list()[-1]
fan_out = ksize * ksize * cnum
stddev = np.sqrt(2. / (fan_in))
# initializer for w used for spectral normalization
w = tf.get_variable(
name + "_w", [ksize, ksize, x.get_shape()[-1], cnum],
initializer=tf.truncated_normal_initializer(stddev=stddev))
x = tf.nn.conv2d(x,
spectral_normed_weight(
w,
update_collection=tf.GraphKeys.UPDATE_OPS,
name=name + "_sn_w"),
strides=[1, stride, stride, 1],
dilations=[1, rate, rate, 1],
padding=padding,
name=name)
return x
def snconv_layer(x, filter_shape, stride, stddev=0.02, update_collection=None):
# Glorot intialization
# For RELU nonlinearity, it's sqrt(2./(n_in)) instead
if stddev is None:
k_h, k_w, input_dim, output_dim = filter_shape
fan_in = k_h * k_w * input_dim
stddev = np.sqrt(2. / (fan_in))
filters = tf.get_variable(
name="weight",
shape=filter_shape,
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev=stddev),
trainable=True)
return tf.nn.conv2d(x, sn.spectral_normed_weight(filters, update_collection=update_collection),
strides=[1, stride, stride, 1], padding="SAME")
def linear_one_hot(input_,
output_size,
num_classes,
name="Linear_one_hot",
stddev=0.01,
scale=1.0,
with_learnable_sn_scale=False,
with_sn=False,
bias_start=0.0,
with_w=False,
update_collection=None,
with_singular_values=False):
with tf.variable_scope(name):
scope_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
tf.get_variable_scope().name)
has_summary = any([('Matrix' in v.op.name) for v in scope_vars])
matrix = tf.get_variable("Matrix", [num_classes, output_size],
tf.float32,
tf.random_normal_initializer(stddev=stddev))
if with_sn:
s = tf.get_variable('s',
shape=[1],
initializer=tf.constant_initializer(scale),
trainable=with_learnable_sn_scale,
dtype=tf.float32)
matrix_bar, sigma = spectral_normed_weight(
matrix, update_collection=update_collection, with_sigma=True)
matrix_bar = s * matrix_bar
embed = tf.nn.embedding_lookup(matrix_bar, input_)
else:
embed = tf.nn.embedding_lookup(matrix, input_)
if not has_summary:
if with_sn:
variable_summaries({'s': s, 'sigma_w': sigma})
variable_summaries({'W': matrix},
with_singular_values=with_singular_values)
else:
variable_summaries({'W': matrix},
with_singular_values=with_singular_values)
if with_w:
return embed, matrix
else:
return embed
def dis_conv(x, cnum, ksize=5, stride=2, name='conv', training=True):
rate = 1
padding = 'SAME'
fan_in = ksize * ksize * x.get_shape().as_list()[-1]
fan_out = ksize * ksize * cnum
stddev = np.sqrt(2. / (fan_in))
w = tf.get_variable(
name + "_w", [ksize, ksize, x.get_shape()[-1], cnum],
initializer=tf.truncated_normal_initializer(stddev=stddev))
x = tf.nn.conv2d(x,
spectral_normed_weight(
w,
update_collection=tf.GraphKeys.UPDATE_OPS,
name=name + "_sn_w"),
strides=[1, stride, stride, 1],
dilations=[1, rate, rate, 1],
padding=padding,
name=name)
return x
def linear(input_, output_size, scope_name="linear",
stddev=0.02, bias_start=0.0, sn_op=None):
with tf.variable_scope(scope_name):
input_ = tf.reshape(
input_,
[-1, np.prod(input_.get_shape().as_list()[1:])])
# output = tf.layers.dense(
# input_,
# output_size)
matrix = tf.get_variable(
"matrix",
[input_.get_shape().as_list()[1], output_size],
tf.float32,
tf.random_normal_initializer(stddev=stddev))
if sn_op is not None:
matrix = spectral_normed_weight(matrix, update_collection=sn_op)
bias = tf.get_variable(
"bias",
[output_size],
initializer=tf.constant_initializer(bias_start))
output = tf.matmul(input_, matrix) + bias
return output
def deconv2d(input_, output_shape,
k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02,
sn_op=None,
name="deconv2d"):
# Code from:
# https://github.com/carpedm20/DCGAN-tensorflow
with tf.variable_scope(name):
# filter : [height, width, output_channels, in_channels]
w = tf.get_variable(
'w',
[k_h, k_w, output_shape[-1], input_.get_shape()[-1]],
initializer=tf.random_normal_initializer(stddev=stddev))
if sn_op is not None:
w = tf.transpose(w, [0, 1, 3, 2])
w = spectral_normed_weight(w, update_collection=sn_op)
w = tf.transpose(w, [0, 1, 3, 2])
try:
deconv = tf.nn.conv2d_transpose(
input_,
w,
output_shape=output_shape,
strides=[1, d_h, d_w, 1])
# Support for verisons of TensorFlow before 0.7.0
except AttributeError:
deconv = tf.nn.deconv2d(
input_,
w,
output_shape=output_shape,
strides=[1, d_h, d_w, 1])
biases = tf.get_variable(
'biases',
[output_shape[-1]],
initializer=tf.constant_initializer(0.0))
deconv = tf.reshape(tf.nn.bias_add(deconv, biases), output_shape)
return deconv
from tensorflow.examples.tutorials.mnist import input_data
from sn import spectral_normed_weight
import timeit
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
SPECTRAL_NORM_UPDATE_OPS = "spectral_norm_update_ops"
x = tf.placeholder(tf.float32, [None, 784])
W = tf.Variable(np.random.normal(size=[784, 10], scale=0.02),
name='W',
dtype=tf.float32)
b = tf.Variable(tf.zeros([10]), name='b', dtype=tf.float32)
W_bar, sigma = spectral_normed_weight(
W,
num_iters=1,
with_sigma=True,
update_collection=SPECTRAL_NORM_UPDATE_OPS)
y = tf.nn.softmax(tf.matmul(x, W_bar) + b)
y_ = tf.placeholder(tf.float32, [None, 10])
cross_entropy = tf.reduce_mean(
-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.001).minimize(cross_entropy)
s, _, _ = tf.svd(W)
s_bar, _, _ = tf.svd(W_bar)
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
def conv2d(input_,
output_dim,
k_h=5,
k_w=5,
d_h=2,
d_w=2,
stddev=0.02,
scale=1.0,
with_learnable_sn_scale=False,
with_sn=False,
name="snconv2d",
update_collection=None,
data_format='NCHW',
with_singular_values=False):
with tf.variable_scope(name):
scope_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
tf.get_variable_scope().name)
has_summary = any([('w' in v.op.name) for v in scope_vars])
out_channel, in_channel = get_in_out_shape(
[output_dim],
input_.get_shape().as_list(), data_format)
strides = get_strides(d_h, d_w, data_format)
w = tf.get_variable(
'w', [k_h, k_w, in_channel, out_channel],
initializer=tf.truncated_normal_initializer(stddev=stddev))
if with_sn:
s = tf.get_variable('s',
shape=[1],
initializer=tf.constant_initializer(scale),
trainable=with_learnable_sn_scale,
dtype=tf.float32)
w_bar, sigma = spectral_normed_weight(
w, update_collection=update_collection, with_sigma=True)
w_bar = s * w_bar
conv = tf.nn.conv2d(input_,
w_bar,
strides=strides,
padding='SAME',
data_format=data_format)
else:
conv = tf.nn.conv2d(input_,
w,
strides=strides,
padding='SAME',
data_format=data_format)
biases = tf.get_variable('biases', [output_dim],
initializer=tf.constant_initializer(0.0))
conv = tf.reshape(
tf.nn.bias_add(conv, biases, data_format=data_format),
conv.get_shape())
if not has_summary:
if with_sn:
variable_summaries({'b': biases, 's': s, 'sigma_w': sigma})
variable_summaries({'W': w},
with_singular_values=with_singular_values)
else:
variable_summaries({'b': biases})
variable_summaries({'W': w},
with_singular_values=with_singular_values)
return conv
def Deconv2D(
name,
input_dim,
output_dim,
filter_size,
inputs,
he_init=True,
weightnorm=None,
spectralnorm=False,
update_collection=None,
biases=True,
gain=1.,
mask_type=None,
):
"""
inputs: tensor of shape (batch size, height, width, input_dim)
returns: tensor of shape (batch size, 2*height, 2*width, output_dim)
"""
with tf.name_scope(name) as scope:
if mask_type != None:
raise Exception('Unsupported configuration')
def uniform(stdev, size):
return np.random.uniform(low=-stdev * np.sqrt(3),
high=stdev * np.sqrt(3),
size=size).astype('float32')
stride = 2
fan_in = input_dim * filter_size**2 / (stride**2)
fan_out = output_dim * filter_size**2
if he_init:
filters_stdev = np.sqrt(4. / (fan_in + fan_out))
else: # Normalized init (Glorot & Bengio)
filters_stdev = np.sqrt(2. / (fan_in + fan_out))
if _weights_stdev is not None:
filter_values = uniform(
_weights_stdev,
(filter_size, filter_size, output_dim, input_dim))
else:
filter_values = uniform(
filters_stdev,
(filter_size, filter_size, output_dim, input_dim))
filter_values *= gain
filters = lib.param(name + '.Filters', filter_values)
if weightnorm == None:
weightnorm = _default_weightnorm
if weightnorm:
norm_values = np.sqrt(
np.sum(np.square(filter_values), axis=(0, 1, 3)))
target_norms = lib.param(name + '.g', norm_values)
with tf.name_scope('weightnorm') as scope:
norms = tf.sqrt(
tf.reduce_sum(tf.square(filters),
reduction_indices=[0, 1, 3]))
filters = filters * tf.expand_dims(target_norms / norms, 1)
if spectralnorm:
filters = spectral_normed_weight(
W=filters, update_collection=update_collection)
inputs = tf.transpose(inputs, [0, 2, 3, 1], name='NCHW_to_NHWC')
# input_shape = tf.shape(inputs)
input_shape = inputs.get_shape()
try: # tf pre-1.0 (top) vs 1.0 (bottom)
output_shape = tf.pack([
int(input_shape[0]), 2 * int(input_shape[1]),
2 * int(input_shape[2]), output_dim
])
except Exception as e:
output_shape = tf.stack([
int(input_shape[0]), 2 * int(input_shape[1]),
2 * int(input_shape[2]), output_dim
])
result = tf.nn.conv2d_transpose(value=inputs,
filter=filters,
output_shape=output_shape,
strides=[1, 2, 2, 1],
padding='SAME')
if biases:
_biases = lib.param(name + '.Biases',
np.zeros(output_dim, dtype='float32'))
result = tf.nn.bias_add(result, _biases)
result = tf.transpose(result, [0, 3, 1, 2], name='NHWC_to_NCHW')
return result
def Conv2D(name, input_dim, output_dim, filter_size, inputs, he_init=True, mask_type=None, stride=1, weightnorm=None,
spectralnorm=False, update_collection = True, biases=True, gain=1.):
"""
inputs: tensor of shape (batch size, num channels, height, width)
mask_type: one of None, 'a', 'b'
returns: tensor of shape (batch size, num channels, height, width)
"""
def scope_has_variables(scope):
return len(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope.name)) > 0
#with tf.name_scope(name) as scope:
with tf.variable_scope(name) as scope:
if scope_has_variables(scope):
scope.reuse_variables()
if mask_type is not None:
mask_type, mask_n_channels = mask_type
mask = np.ones(
(filter_size, filter_size, input_dim, output_dim),
dtype='float32'
)
center = filter_size // 2
# Mask out future locations
# filter shape is (height, width, input channels, output channels)
mask[center+1:, :, :, :] = 0.
mask[center, center+1:, :, :] = 0.
# Mask out future channels
for i in xrange(mask_n_channels):
for j in xrange(mask_n_channels):
if (mask_type=='a' and i >= j) or (mask_type=='b' and i > j):
mask[
center,
center,
i::mask_n_channels,
j::mask_n_channels
] = 0.
def uniform(stdev, size):
return np.random.uniform(
low=-stdev * np.sqrt(3),
high=stdev * np.sqrt(3),
size=size
).astype('float32')
fan_in = input_dim * filter_size**2
fan_out = output_dim * filter_size**2 / (stride**2)
if mask_type is not None: # only approximately correct
fan_in /= 2.
fan_out /= 2.
if he_init:
filters_stdev = np.sqrt(4./(fan_in+fan_out))
else: # Normalized init (Glorot & Bengio)
filters_stdev = np.sqrt(2./(fan_in+fan_out))
if _weights_stdev is not None:
filter_values = uniform(
_weights_stdev,
(filter_size, filter_size, input_dim, output_dim)
)
else:
filter_values = uniform(
filters_stdev,
(filter_size, filter_size, input_dim, output_dim)
)
# print "WARNING IGNORING GAIN"
filter_values *= gain
filters = lib.param(name+'.Filters', filter_values)
if weightnorm==None:
weightnorm = _default_weightnorm
if weightnorm:
norm_values = np.sqrt(np.sum(np.square(filter_values), axis=(0,1,2)))
target_norms = lib.param(
name + '.g',
norm_values
)
with tf.name_scope('weightnorm') as scope:
norms = tf.sqrt(tf.reduce_sum(tf.square(filters), reduction_indices=[0,1,2]))
filters = filters * (target_norms / norms)
if spectralnorm:
filters = spectral_normed_weight(W = filters, update_collection=update_collection)
if mask_type is not None:
with tf.name_scope('filter_mask'):
filters = filters * mask
result = tf.nn.conv2d(
input=inputs,
filter=filters,
strides=[1, 1, stride, stride],
padding='SAME',
data_format='NCHW'
)
if biases:
_biases = lib.param(
name+'.Biases',
np.zeros(output_dim, dtype='float32')
)
result = tf.nn.bias_add(result, _biases, data_format='NCHW')
return result
def conv2d(inputs,
num_outputs,
kernel_size,
sn,
stride=1,
rate=1,
data_format='NCHW',
activation_fn=tf.nn.relu,
normalizer_fn=None,
normalizer_params=None,
weights_regularizer=None,
weights_initializer=ly.xavier_initializer(),
biases_initializer=init_ops.zeros_initializer(),
biases_regularizer=None,
reuse=None,
scope=None):
assert data_format == 'NCHW'
assert rate == 1
if data_format == 'NCHW':
channel_axis = 1
stride = [1, 1, stride, stride]
rate = [1, 1, rate, rate]
else:
channel_axis = 3
stride = [1, stride, stride, 1]
rate = [1, rate, rate, 1]
input_dim = inputs.get_shape().as_list()[channel_axis]
with tf.variable_scope(scope, 'Conv', [inputs], reuse=reuse) as sc:
inputs = tf.convert_to_tensor(inputs)
weights = tf.get_variable(name="weights",
shape=(kernel_size, kernel_size, input_dim,
num_outputs),
initializer=weights_initializer,
regularizer=weights_regularizer,
trainable=True,
dtype=inputs.dtype.base_dtype)
# Spectral Normalization
if sn:
weights = spectral_normed_weight(
weights,
num_iters=1,
update_collection=Config.SPECTRAL_NORM_UPDATE_OPS)
conv_out = tf.nn.conv2d(inputs,
weights,
strides=stride,
padding='SAME',
data_format=data_format)
if biases_initializer is not None:
biases = tf.get_variable(name='biases',
shape=(1, num_outputs, 1, 1),
initializer=biases_initializer,
regularizer=biases_regularizer,
trainable=True,
dtype=inputs.dtype.base_dtype)
conv_out += biases
if normalizer_fn is not None:
normalizer_params = normalizer_params or {}
conv_out = normalizer_fn(conv_out,
activation_fn=None,
**normalizer_params)
if activation_fn is not None:
conv_out = activation_fn(conv_out)
return conv_out
def Linear(name,
input_dim,
output_dim,
inputs,
biases=True,
initialization=None,
weightnorm=None,
spectralnorm=False,
update_collection=None,
gain=1.):
"""
initialization: None, `lecun`, 'glorot', `he`, 'glorot_he', `orthogonal`, `("uniform", range)`
"""
with tf.variable_scope(name) as scope:
if scope_has_variables(scope):
scope.reuse_variables()
def uniform(stdev, size):
if _weights_stdev is not None:
stdev = _weights_stdev
return np.random.uniform(low=-stdev * np.sqrt(3),
high=stdev * np.sqrt(3),
size=size).astype('float32')
if initialization == 'lecun': # and input_dim != output_dim):
# disabling orth. init for now because it's too slow
weight_values = uniform(np.sqrt(1. / input_dim),
(input_dim, output_dim))
elif initialization == 'glorot' or (initialization == None):
weight_values = uniform(np.sqrt(2. / (input_dim + output_dim)),
(input_dim, output_dim))
elif initialization == 'he':
weight_values = uniform(np.sqrt(2. / input_dim),
(input_dim, output_dim))
elif initialization == 'glorot_he':
weight_values = uniform(np.sqrt(4. / (input_dim + output_dim)),
(input_dim, output_dim))
elif initialization == 'orthogonal' or \
(initialization == None and input_dim == output_dim):
# From lasagne
def sample(shape):
if len(shape) < 2:
raise RuntimeError("Only shapes of length 2 or more are "
"supported.")
flat_shape = (shape[0], np.prod(shape[1:]))
# TODO: why normal and not uniform?
a = np.random.normal(0.0, 1.0, flat_shape)
u, _, v = np.linalg.svd(a, full_matrices=False)
# pick the one with the correct shape
q = u if u.shape == flat_shape else v
q = q.reshape(shape)
return q.astype('float32')
weight_values = sample((input_dim, output_dim))
elif initialization[0] == 'uniform':
weight_values = np.random.uniform(
low=-initialization[1],
high=initialization[1],
size=(input_dim, output_dim)).astype('float32')
else:
raise Exception('Invalid initialization!')
weight_values *= gain
weight = lib.param(name + '.W', weight_values)
if weightnorm == None:
weightnorm = _default_weightnorm
if weightnorm:
norm_values = np.sqrt(np.sum(np.square(weight_values), axis=0))
# norm_values = np.linalg.norm(weight_values, axis=0)
target_norms = lib.param(name + '.g', norm_values)
with tf.name_scope('weightnorm') as scope:
norms = tf.sqrt(
tf.reduce_sum(tf.square(weight), reduction_indices=[0]))
weight = weight * (target_norms / norms)
if spectralnorm:
weight = spectral_normed_weight(
weight, update_collection=update_collection)
# if 'Discriminator' in name:
# print "WARNING weight constraint on {}".format(name)
# weight = tf.nn.softsign(10.*weight)*.1
if inputs.get_shape().ndims == 2:
result = tf.matmul(inputs, weight)
else:
reshaped_inputs = tf.reshape(inputs, [-1, input_dim])
result = tf.matmul(reshaped_inputs, weight)
result = tf.reshape(
result,
tf.pack(tf.unpack(tf.shape(inputs))[:-1] + [output_dim]))
if biases:
result = tf.nn.bias_add(
result,
lib.param(name + '.b', np.zeros((output_dim, ),
dtype='float32')))
return result
def deconv2d(input_,
output_shape,
k_h=5,
k_w=5,
d_h=2,
d_w=2,
stddev=0.02,
scale=1.0,
with_learnable_sn_scale=False,
with_sn=False,
name="deconv2d",
with_w=False,
update_collection=None,
data_format='NCHW',
with_singular_values=False):
with tf.variable_scope(name):
scope_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
tf.get_variable_scope().name)
has_summary = any([('w' in v.op.name) for v in scope_vars])
out_channel, in_channel = get_in_out_shape(
output_shape,
input_.get_shape().as_list(), data_format)
strides = get_strides(d_h, d_w, data_format)
# filter : [height, width, output_channels, in_channels]
w = tf.get_variable(
'w', [k_h, k_w, out_channel, in_channel],
initializer=tf.random_normal_initializer(stddev=stddev))
if with_sn:
s = tf.get_variable('s',
shape=[1],
initializer=tf.constant_initializer(scale),
trainable=with_learnable_sn_scale,
dtype=tf.float32)
w_bar, sigma = spectral_normed_weight(
w, update_collection=update_collection, with_sigma=True)
w_bar = s * w_bar
deconv = tf.nn.conv2d_transpose(input_,
w_bar,
output_shape=output_shape,
strides=strides,
data_format=data_format)
else:
deconv = tf.nn.conv2d_transpose(input_,
w,
output_shape=output_shape,
strides=strides,
data_format=data_format)
biases = tf.get_variable('biases', [out_channel],
initializer=tf.constant_initializer(0.0))
deconv = tf.reshape(
tf.nn.bias_add(deconv, biases, data_format=data_format),
deconv.get_shape())
if not has_summary:
if with_sn:
variable_summaries({'b': biases, 's': s, 'sigma_w': sigma})
variable_summaries({'W': w},
with_singular_values=with_singular_values)
else:
variable_summaries({'b': biases})
variable_summaries({'W': w},
with_singular_values=with_singular_values)
if with_w:
return deconv, w, biases
else:
return deconv
