def deconv2d(input_, output_shape,
k_h=4, k_w=4, d_h=2, d_w=2, stddev=None,
name="deconv2d", spectral_normed=False, update_collection=tf.GraphKeys.UPDATE_OPS, with_w=False, padding="SAME"):
# Glorot initialization
# For RELU nonlinearity, it's sqrt(2./(n_in)) instead
fan_in = k_h * k_w * input_.get_shape().as_list()[-1]
fan_out = k_h * k_w * output_shape[-1]
if stddev is None:
stddev = np.sqrt(2. / (fan_in))
with tf.variable_scope(name) as scope:
if scope_has_variables(scope):
scope.reuse_variables()
# filter : [height, width, output_channels, in_channels]
w = tf.get_variable("w", [k_h, k_w, output_shape[-1], input_.get_shape()[-1]],
initializer=tf.truncated_normal_initializer(stddev=stddev))
if spectral_normed:
deconv = tf.nn.conv2d_transpose(input_, spectral_normed_weight(w, update_collection=update_collection),
output_shape=output_shape,
strides=[1, d_h, d_w, 1], padding=padding)
else:
deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape,
strides=[1, d_h, d_w, 1], padding=padding)
biases = tf.get_variable("b", [output_shape[-1]], initializer=tf.constant_initializer(0))
deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape())
if with_w:
return deconv, w, biases
else:
return deconv
def linear(input_, output_size, name="linear", spectral_normed=False, update_collection=tf.GraphKeys.UPDATE_OPS, stddev=None, bias_start=0.0, with_biases=True,
with_w=False):
shape = input_.get_shape().as_list()
if stddev is None:
stddev = np.sqrt(1. / (shape[1]))
with tf.variable_scope(name) as scope:
if scope_has_variables(scope):
scope.reuse_variables()
weight = tf.get_variable("w", [shape[1], output_size], tf.float32,
tf.truncated_normal_initializer(stddev=stddev))
if with_biases:
bias = tf.get_variable("b", [output_size],
initializer=tf.constant_initializer(bias_start))
if spectral_normed:
mul = tf.matmul(input_, spectral_normed_weight(weight, update_collection=update_collection))
else:
mul = tf.matmul(input_, weight)
if with_w:
if with_biases:
return mul + bias, weight, bias
else:
return mul, weight, None
else:
if with_biases:
return mul + bias
else:
return mul
def linear(input_,
output_size,
stddev=0.02,
use_spectral_norm=False,
name='Linear'):
shape = input_.get_shape().as_list()
with tf.variable_scope(name):
matrix = tf.get_variable("Matrix", [shape[1], output_size],
tf.float32,
initializer=tf.random_normal_initializer(
stddev=stddev, seed=4285))
variable_summaries(matrix, 'weights')
b = tf.get_variable('b', [output_size],
initializer=tf.constant_initializer(0.02))
variable_summaries(b, 'biases')
# if not tf.get_variable_scope().reuse:
# tf.histogram_summary(matrix.name, matrix)
if use_spectral_norm:
mul = tf.matmul(
input_,
spectral_normed_weight(
matrix, update_collection=SPECTRAL_NORM_UPDATE_OPS))
else:
mul = tf.matmul(input_, matrix)
pre_act = mul + b
variable_summaries(pre_act, 'pre_activations')
return pre_act
def conv2d(input_, output_dim,
k_h=4, k_w=4, d_h=2, d_w=2, stddev=None,
name="conv2d", spectral_normed=False, update_collection=None, with_w=False, padding="SAME"):
# Glorot intialization
# For RELU nonlinearity, it's sqrt(2./(n_in)) instead
fan_in = k_h * k_w * input_.get_shape().as_list()[-1]
fan_out = k_h * k_w * output_dim
if stddev is None:
stddev = np.sqrt(2. / (fan_in))
with tf.variable_scope(name) as scope:
if scope_has_variables(scope):
scope.reuse_variables()
w = tf.get_variable("w", [k_h, k_w, input_.get_shape()[-1], output_dim],
initializer=tf.truncated_normal_initializer(stddev=stddev))
if spectral_normed:
conv = tf.nn.conv2d(input_, spectral_normed_weight(w, update_collection=update_collection),
strides=[1, d_h, d_w, 1], padding=padding)
else:
conv = tf.nn.conv2d(input_, w, strides=[1, d_h, d_w, 1], padding=padding)
biases = tf.get_variable("b", [output_dim], initializer=tf.constant_initializer(0.0))
conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape())
if with_w:
return conv, w, biases
else:
return conv
def deconv2d(input_,
output_shape,
k_h=3,
k_w=3,
d_h=2,
d_w=2,
stddev=0.02,
padding='SAME',
use_spectral_norm=False,
name="deconv2d"):
with tf.variable_scope(name):
# filter : [height, width, output_channels, in_channels]
# TODO: 2nd param should be k_w?
w = tf.get_variable(
'w', [k_h, k_h, output_shape[-1],
input_.get_shape()[-1]],
initializer=tf.random_normal_initializer(stddev=stddev, seed=4285))
if use_spectral_norm:
w_bar = spectral_normed_weight(
w, update_collection=SPECTRAL_NORM_UPDATE_OPS)
w = w_bar
# if not tf.get_variable_scope().reuse:
# tf.summary.histogram(w.name, w)
return tf.nn.conv2d_transpose(input_,
w,
output_shape=output_shape,
strides=[1, d_h, d_w, 1],
padding=padding)
def conv2d(input_,
output_dim,
k_h=3,
k_w=3,
d_h=2,
d_w=2,
stddev=0.01,
padding='SAME',
use_spectral_norm=False,
name="conv2d"):
with tf.variable_scope(name):
in_channels = input_.get_shape()[-1]
out_channels = output_dim
w = tf.get_variable(
'w', [k_h, k_w, in_channels, out_channels],
initializer=tf.truncated_normal_initializer(stddev=stddev))
if use_spectral_norm:
w_bar = spectral_normed_weight(
w, update_collection=SPECTRAL_NORM_UPDATE_OPS)
w = w_bar
b = tf.get_variable('b', [out_channels],
initializer=tf.constant_initializer(0.01))
# if not tf.get_variable_scope().reuse:
# tf.summary.histogram(w.name, w)
conv = tf.nn.conv2d(input_,
w,
strides=[1, d_h, d_w, 1],
padding=padding)
conv = tf.nn.bias_add(conv, b)
return conv
def linear(input_, output_size, name="linear",
spectral_normed=False, update_collection=None, stddev=None,
bias_start=0.0, with_biases=False, with_w=False,
mhe=False, net_type='d'):
shape = input_.get_shape().as_list()
if stddev is None:
stddev = np.sqrt(1. / (shape[1]))
with tf.variable_scope(name) as scope:
if scope_has_variables(scope):
scope.reuse_variables()
weight = tf.get_variable("w", [shape[1], output_size], tf.float32,
tf.truncated_normal_initializer(stddev=stddev))
if with_biases:
bias = tf.get_variable("b", [output_size],
initializer=tf.constant_initializer(bias_start))
if spectral_normed:
mul = tf.matmul(input_, spectral_normed_weight(weight, update_collection=update_collection))
else:
mul = tf.matmul(input_, weight)
########## mhe ###########
if mhe:
eps = 1e-4
filt = weight
filt_num = filt.get_shape().as_list()[-1]
filt = tf.concat([filt, -filt], axis=0)
filt_norm = tf.sqrt(tf.reduce_sum(filt*filt, [0], keep_dims=True)+eps)
filt /= filt_norm
inner_pro = tf.matmul(tf.transpose(filt), filt)
cross_terms = 2.0 - 2.0*inner_pro
cross_terms = tf.matrix_band_part(cross_terms, 0, -1) * (1.0 - np.eye(filt_num))
loss = -1e-7 * tf.reduce_mean( tf.log(cross_terms + eps))
if net_type == 'g':
tf.add_to_collection('g_mhe_loss', loss)
elif net_type == 'd':
tf.add_to_collection('d_mhe_loss', loss)
else:
raise
############################
if with_w:
if with_biases:
return mul + bias, weight, bias
else:
return mul, weight, None
else:
if with_biases:
return mul + bias
else:
return mul
def deconv2d(input_, output_shape,
k_h=4, k_w=4, d_h=2, d_w=2, stddev=None,
name="deconv2d", spectral_normed=False, update_collection=None, with_w=False, padding="SAME",
mhe=False, net_type='g'):
# Glorot initialization
# For RELU nonlinearity, it's sqrt(2./(n_in)) instead
fan_in = k_h * k_w * input_.get_shape().as_list()[-1]
fan_out = k_h * k_w * output_shape[-1]
if stddev is None:
stddev = np.sqrt(2. / (fan_in))
with tf.variable_scope(name) as scope:
if scope_has_variables(scope):
scope.reuse_variables()
# filter : [height, width, output_channels, in_channels]
w = tf.get_variable("w", [k_h, k_w, output_shape[-1], input_.get_shape()[-1]],
initializer=tf.truncated_normal_initializer(stddev=stddev))
if spectral_normed:
deconv = tf.nn.conv2d_transpose(input_, spectral_normed_weight(w, update_collection=update_collection),
output_shape=output_shape,
strides=[1, d_h, d_w, 1], padding=padding)
else:
deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape,
strides=[1, d_h, d_w, 1], padding=padding)
########## mhe ###########
if mhe:
eps = 1e-4
filt = w
filt_num = input_.get_shape().as_list()[-1]
filt = tf.reshape(filt, [-1, filt_num])
filt = tf.concat([filt, -filt], axis=0)
filt_norm = tf.sqrt(tf.reduce_sum(filt*filt, [0], keep_dims=True)+eps)
filt /= filt_norm
inner_pro = tf.matmul(tf.transpose(filt), filt)
cross_terms = 2.0 - 2.0*inner_pro
cross_terms = tf.matrix_band_part(cross_terms, 0, -1) * (1.0 - np.eye(filt_num))
loss = -1e-7 * tf.reduce_mean( tf.log(cross_terms + eps))
if net_type == 'g':
tf.add_to_collection('g_mhe_loss', loss)
else:
raise
############################
biases = tf.get_variable("b", [output_shape[-1]], initializer=tf.constant_initializer(0))
deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape())
if with_w:
return deconv, w, biases
else:
return deconv
import numpy as np
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
from libs.sn import spectral_normed_weight
import timeit
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
SPECTRAL_NORM_UPDATE_OPS = "spectral_norm_update_ops"
x = tf.placeholder(tf.float32, [None, 784])
W = tf.Variable(np.random.normal(size=[784, 10], scale=0.02), name='W', dtype=tf.float32)
b = tf.Variable(tf.zeros([10]), name='b', dtype=tf.float32)
W_bar, sigma = spectral_normed_weight(W, num_iters=1, with_sigma=True, update_collection=SPECTRAL_NORM_UPDATE_OPS)
y = tf.nn.softmax(tf.matmul(x, W_bar) + b)
y_ = tf.placeholder(tf.float32, [None, 10])
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.001).minimize(cross_entropy)
s, _, _ = tf.svd(W)
s_bar, _, _ = tf.svd(W_bar)
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
update_ops = tf.get_collection(SPECTRAL_NORM_UPDATE_OPS)
for _ in range(1000):
start = timeit.default_timer()
batch_xs, batch_ys = mnist.train.next_batch(100)
sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})
