def conv2d(inputs, out_dim, k_size, strides, padding="SAME", w_init=None, use_bias=True, spectral_normed=True, name="conv2d", ): with tf.variable_scope(name): w = tf.get_variable("w", shape=[k_size, k_size, inputs.get_shape()[-1], out_dim], dtype=tf.float32, initializer=w_init ) if spectral_normed: w = spectral_normed_weight(w) conv = tf.nn.conv2d(inputs, w, strides=[1, strides, strides, 1], padding=padding.upper()) if use_bias: biases = tf.get_variable("b", [out_dim], initializer=tf.constant_initializer(0.0)) conv = tf.nn.bias_add(conv, biases, name="conv_add_b") return conv
def conv2d(input_, output_dim, k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02, sn_op=None, name="conv2d"): # Code from: # https://github.com/carpedm20/DCGAN-tensorflow with tf.variable_scope(name): w = tf.get_variable( 'w', [k_h, k_w, input_.get_shape()[-1], output_dim], initializer=tf.truncated_normal_initializer(stddev=stddev)) if sn_op is not None: w = spectral_normed_weight(w, update_collection=sn_op) conv = tf.nn.conv2d( input_, w, strides=[1, d_h, d_w, 1], padding='SAME') biases = tf.get_variable( 'biases', [output_dim], initializer=tf.constant_initializer(0.0)) conv = tf.reshape( tf.nn.bias_add(conv, biases), [-1] + conv.get_shape().as_list()[1:]) return conv
def linear(inputs, out_dim, w_init=None, activation=None, use_bias=True, bias_start=0.0, spectral_normed=False, name="linear", ): with tf.variable_scope(name): w = tf.get_variable("w", shape=[ inputs.get_shape()[-1], out_dim ], dtype=tf.float32, initializer=w_init ) if spectral_normed: w = spectral_normed_weight(w) mul = tf.matmul(inputs, w, name='linear_mul') if use_bias: bias = tf.get_variable("b", [out_dim], initializer=tf.constant_initializer(bias_start)) mul = tf.nn.bias_add(mul, bias, name="mul_add_b") if not (activation is None): mul = activation(mul) return mul
def conv2d(input_, output_dim, k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02, spectral_norm=False, name="conv2d"): with tf.variable_scope(name): w = tf.get_variable( 'w', [k_h, k_w, input_.get_shape()[-1], output_dim], initializer=tf.truncated_normal_initializer(stddev=stddev)) if spectral_norm: w = sn.spectral_normed_weight(w, update_collection=None) conv = tf.nn.conv2d(input_, w, strides=[1, d_h, d_w, 1], padding='SAME') biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0)) conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape()) return conv
def embed_labels(inputs, num_classes, output_dim, sn, weight_decay_rate=1e-5, reuse=None, scope=None): # TODO move regularizer definitions to model weights_regularizer = ly.l2_regularizer(weight_decay_rate) with tf.variable_scope(scope, 'embedding', [inputs], reuse=reuse) as sc: inputs = tf.convert_to_tensor(inputs) weights = tf.get_variable( name="weights", shape=(num_classes, output_dim), initializer=init_ops.random_normal_initializer) # Spectral Normalization if sn: weights = spectral_normed_weight( weights, num_iters=1, update_collection=Config.SPECTRAL_NORM_UPDATE_OPS) embed_out = tf.nn.embedding_lookup(weights, inputs) return embed_out
def sndilated_conv_layer(x, filter_shape, dilation, stddev=0.02, update_collection=None): filters = tf.get_variable( name='weight', shape=filter_shape, dtype=tf.float32, initializer=tf.truncated_normal_initializer(stddev=stddev), trainable=True) return tf.nn.atrous_conv2d(x, sn.spectral_normed_weight(filters, update_collection=update_collection), dilation, padding='SAME')
def fully_connected(inputs, num_outputs, sn, activation_fn=None, normalizer_fn=None, normalizer_params=None, weights_initializer=ly.xavier_initializer(), weight_decay_rate=1e-6, biases_initializer=init_ops.zeros_initializer(), biases_regularizer=None, reuse=None, scope=None): # TODO move regularizer definitions to model weights_regularizer = ly.l2_regularizer(weight_decay_rate) input_dim = inputs.get_shape().as_list()[1] with tf.variable_scope(scope, 'fully_connected', [inputs], reuse=reuse) as sc: inputs = tf.convert_to_tensor(inputs) weights = tf.get_variable(name="weights", shape=(input_dim, num_outputs), initializer=weights_initializer, regularizer=weights_regularizer, trainable=True, dtype=inputs.dtype.base_dtype) # Spectral Normalization if sn: weights = spectral_normed_weight( weights, num_iters=1, update_collection=Config.SPECTRAL_NORM_UPDATE_OPS) linear_out = tf.matmul(inputs, weights) if biases_initializer is not None: biases = tf.get_variable(name="biases", shape=(num_outputs, ), initializer=biases_initializer, regularizer=biases_regularizer, trainable=True, dtype=inputs.dtype.base_dtype) linear_out = tf.nn.bias_add(linear_out, biases) # Apply normalizer function / layer. if normalizer_fn is not None: normalizer_params = normalizer_params or {} linear_out = normalizer_fn(linear_out, activation_fn=None, **normalizer_params) if activation_fn is not None: linear_out = activation_fn(linear_out) return linear_out
def linear(input, output_dim, scope='linear', stddev=0.01,use_SN=False, update_collection=None): norm = tf.random_normal_initializer(stddev=stddev) const = tf.constant_initializer(0.0) with tf.variable_scope(scope): w = tf.get_variable('weights', [input.get_shape()[1], output_dim], initializer=norm) b = tf.get_variable('biases', [output_dim], initializer=const) if use_SN: print('using SN') return tf.matmul(input,spectral_normed_weight(w, update_collection=update_collection))+b else: return tf.matmul(input, w) + b
def linear(input_, output_size, name="Linear", stddev=0.01, scale=1.0, with_learnable_sn_scale=False, with_sn=False, bias_start=0.0, with_w=False, update_collection=None, with_singular_values=False): shape = input_.get_shape().as_list() with tf.variable_scope(name): scope_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, tf.get_variable_scope().name) has_summary = any([('Matrix' in v.op.name) for v in scope_vars]) matrix = tf.get_variable("Matrix", [shape[1], output_size], tf.float32, tf.random_normal_initializer(stddev=stddev)) if with_sn: s = tf.get_variable('s', shape=[1], initializer=tf.constant_initializer(scale), trainable=with_learnable_sn_scale, dtype=tf.float32) matrix_bar, sigma = spectral_normed_weight( matrix, update_collection=update_collection, with_sigma=True) matrix_bar = s * matrix_bar mul = tf.matmul(input_, matrix_bar) else: mul = tf.matmul(input_, matrix) bias = tf.get_variable("bias", [output_size], initializer=tf.constant_initializer(bias_start)) if not has_summary: if with_sn: variable_summaries({'b': bias, 's': s, 'sigma_w': sigma}) variable_summaries({'W': matrix}, with_singular_values=with_singular_values) else: variable_summaries({'b': bias}) variable_summaries({'W': matrix}, with_singular_values=with_singular_values) if with_w: return mul + bias, matrix, bias else: return mul + bias
def gen_snconv(x, cnum, ksize, stride=1, rate=1, name='conv', padding='SAME', activation=tf.nn.relu, training=True): """Define spectral normalization conv for discriminator. Args: x: Input. cnum: Channel number. ksize: Kernel size. Stride: Convolution stride. Rate: Rate for or dilated conv. name: Name of layers. padding: Default to SYMMETRIC. activation: Activation function after convolution. training: If current graph is for training or inference, used for bn. Returns: tf.Tensor: output """ assert padding in ['SYMMETRIC', 'SAME', 'REFELECT'] if padding == 'SYMMETRIC' or padding == 'REFELECT': p = int(rate * (ksize - 1) / 2) x = tf.pad(x, [[0, 0], [p, p], [p, p], [0, 0]], mode=padding) padding = 'VALID' fan_in = ksize * ksize * x.get_shape().as_list()[-1] fan_out = ksize * ksize * cnum stddev = np.sqrt(2. / (fan_in)) # initializer for w used for spectral normalization w = tf.get_variable( name + "_w", [ksize, ksize, x.get_shape()[-1], cnum], initializer=tf.truncated_normal_initializer(stddev=stddev)) x = tf.nn.conv2d(x, spectral_normed_weight( w, update_collection=tf.GraphKeys.UPDATE_OPS, name=name + "_sn_w"), strides=[1, stride, stride, 1], dilations=[1, rate, rate, 1], padding=padding, name=name) return x
def snconv_layer(x, filter_shape, stride, stddev=0.02, update_collection=None): # Glorot intialization # For RELU nonlinearity, it's sqrt(2./(n_in)) instead if stddev is None: k_h, k_w, input_dim, output_dim = filter_shape fan_in = k_h * k_w * input_dim stddev = np.sqrt(2. / (fan_in)) filters = tf.get_variable( name="weight", shape=filter_shape, dtype=tf.float32, initializer=tf.truncated_normal_initializer(stddev=stddev), trainable=True) return tf.nn.conv2d(x, sn.spectral_normed_weight(filters, update_collection=update_collection), strides=[1, stride, stride, 1], padding="SAME")
def linear_one_hot(input_, output_size, num_classes, name="Linear_one_hot", stddev=0.01, scale=1.0, with_learnable_sn_scale=False, with_sn=False, bias_start=0.0, with_w=False, update_collection=None, with_singular_values=False): with tf.variable_scope(name): scope_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, tf.get_variable_scope().name) has_summary = any([('Matrix' in v.op.name) for v in scope_vars]) matrix = tf.get_variable("Matrix", [num_classes, output_size], tf.float32, tf.random_normal_initializer(stddev=stddev)) if with_sn: s = tf.get_variable('s', shape=[1], initializer=tf.constant_initializer(scale), trainable=with_learnable_sn_scale, dtype=tf.float32) matrix_bar, sigma = spectral_normed_weight( matrix, update_collection=update_collection, with_sigma=True) matrix_bar = s * matrix_bar embed = tf.nn.embedding_lookup(matrix_bar, input_) else: embed = tf.nn.embedding_lookup(matrix, input_) if not has_summary: if with_sn: variable_summaries({'s': s, 'sigma_w': sigma}) variable_summaries({'W': matrix}, with_singular_values=with_singular_values) else: variable_summaries({'W': matrix}, with_singular_values=with_singular_values) if with_w: return embed, matrix else: return embed
def dis_conv(x, cnum, ksize=5, stride=2, name='conv', training=True): rate = 1 padding = 'SAME' fan_in = ksize * ksize * x.get_shape().as_list()[-1] fan_out = ksize * ksize * cnum stddev = np.sqrt(2. / (fan_in)) w = tf.get_variable( name + "_w", [ksize, ksize, x.get_shape()[-1], cnum], initializer=tf.truncated_normal_initializer(stddev=stddev)) x = tf.nn.conv2d(x, spectral_normed_weight( w, update_collection=tf.GraphKeys.UPDATE_OPS, name=name + "_sn_w"), strides=[1, stride, stride, 1], dilations=[1, rate, rate, 1], padding=padding, name=name) return x
def linear(input_, output_size, scope_name="linear", stddev=0.02, bias_start=0.0, sn_op=None): with tf.variable_scope(scope_name): input_ = tf.reshape( input_, [-1, np.prod(input_.get_shape().as_list()[1:])]) # output = tf.layers.dense( # input_, # output_size) matrix = tf.get_variable( "matrix", [input_.get_shape().as_list()[1], output_size], tf.float32, tf.random_normal_initializer(stddev=stddev)) if sn_op is not None: matrix = spectral_normed_weight(matrix, update_collection=sn_op) bias = tf.get_variable( "bias", [output_size], initializer=tf.constant_initializer(bias_start)) output = tf.matmul(input_, matrix) + bias return output
def deconv2d(input_, output_shape, k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02, sn_op=None, name="deconv2d"): # Code from: # https://github.com/carpedm20/DCGAN-tensorflow with tf.variable_scope(name): # filter : [height, width, output_channels, in_channels] w = tf.get_variable( 'w', [k_h, k_w, output_shape[-1], input_.get_shape()[-1]], initializer=tf.random_normal_initializer(stddev=stddev)) if sn_op is not None: w = tf.transpose(w, [0, 1, 3, 2]) w = spectral_normed_weight(w, update_collection=sn_op) w = tf.transpose(w, [0, 1, 3, 2]) try: deconv = tf.nn.conv2d_transpose( input_, w, output_shape=output_shape, strides=[1, d_h, d_w, 1]) # Support for verisons of TensorFlow before 0.7.0 except AttributeError: deconv = tf.nn.deconv2d( input_, w, output_shape=output_shape, strides=[1, d_h, d_w, 1]) biases = tf.get_variable( 'biases', [output_shape[-1]], initializer=tf.constant_initializer(0.0)) deconv = tf.reshape(tf.nn.bias_add(deconv, biases), output_shape) return deconv
from tensorflow.examples.tutorials.mnist import input_data from sn import spectral_normed_weight import timeit mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) SPECTRAL_NORM_UPDATE_OPS = "spectral_norm_update_ops" x = tf.placeholder(tf.float32, [None, 784]) W = tf.Variable(np.random.normal(size=[784, 10], scale=0.02), name='W', dtype=tf.float32) b = tf.Variable(tf.zeros([10]), name='b', dtype=tf.float32) W_bar, sigma = spectral_normed_weight( W, num_iters=1, with_sigma=True, update_collection=SPECTRAL_NORM_UPDATE_OPS) y = tf.nn.softmax(tf.matmul(x, W_bar) + b) y_ = tf.placeholder(tf.float32, [None, 10]) cross_entropy = tf.reduce_mean( -tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1])) train_step = tf.train.GradientDescentOptimizer(0.001).minimize(cross_entropy) s, _, _ = tf.svd(W) s_bar, _, _ = tf.svd(W_bar) sess = tf.InteractiveSession() tf.global_variables_initializer().run()
def conv2d(input_, output_dim, k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02, scale=1.0, with_learnable_sn_scale=False, with_sn=False, name="snconv2d", update_collection=None, data_format='NCHW', with_singular_values=False): with tf.variable_scope(name): scope_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, tf.get_variable_scope().name) has_summary = any([('w' in v.op.name) for v in scope_vars]) out_channel, in_channel = get_in_out_shape( [output_dim], input_.get_shape().as_list(), data_format) strides = get_strides(d_h, d_w, data_format) w = tf.get_variable( 'w', [k_h, k_w, in_channel, out_channel], initializer=tf.truncated_normal_initializer(stddev=stddev)) if with_sn: s = tf.get_variable('s', shape=[1], initializer=tf.constant_initializer(scale), trainable=with_learnable_sn_scale, dtype=tf.float32) w_bar, sigma = spectral_normed_weight( w, update_collection=update_collection, with_sigma=True) w_bar = s * w_bar conv = tf.nn.conv2d(input_, w_bar, strides=strides, padding='SAME', data_format=data_format) else: conv = tf.nn.conv2d(input_, w, strides=strides, padding='SAME', data_format=data_format) biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0)) conv = tf.reshape( tf.nn.bias_add(conv, biases, data_format=data_format), conv.get_shape()) if not has_summary: if with_sn: variable_summaries({'b': biases, 's': s, 'sigma_w': sigma}) variable_summaries({'W': w}, with_singular_values=with_singular_values) else: variable_summaries({'b': biases}) variable_summaries({'W': w}, with_singular_values=with_singular_values) return conv
def Deconv2D( name, input_dim, output_dim, filter_size, inputs, he_init=True, weightnorm=None, spectralnorm=False, update_collection=None, biases=True, gain=1., mask_type=None, ): """ inputs: tensor of shape (batch size, height, width, input_dim) returns: tensor of shape (batch size, 2*height, 2*width, output_dim) """ with tf.name_scope(name) as scope: if mask_type != None: raise Exception('Unsupported configuration') def uniform(stdev, size): return np.random.uniform(low=-stdev * np.sqrt(3), high=stdev * np.sqrt(3), size=size).astype('float32') stride = 2 fan_in = input_dim * filter_size**2 / (stride**2) fan_out = output_dim * filter_size**2 if he_init: filters_stdev = np.sqrt(4. / (fan_in + fan_out)) else: # Normalized init (Glorot & Bengio) filters_stdev = np.sqrt(2. / (fan_in + fan_out)) if _weights_stdev is not None: filter_values = uniform( _weights_stdev, (filter_size, filter_size, output_dim, input_dim)) else: filter_values = uniform( filters_stdev, (filter_size, filter_size, output_dim, input_dim)) filter_values *= gain filters = lib.param(name + '.Filters', filter_values) if weightnorm == None: weightnorm = _default_weightnorm if weightnorm: norm_values = np.sqrt( np.sum(np.square(filter_values), axis=(0, 1, 3))) target_norms = lib.param(name + '.g', norm_values) with tf.name_scope('weightnorm') as scope: norms = tf.sqrt( tf.reduce_sum(tf.square(filters), reduction_indices=[0, 1, 3])) filters = filters * tf.expand_dims(target_norms / norms, 1) if spectralnorm: filters = spectral_normed_weight( W=filters, update_collection=update_collection) inputs = tf.transpose(inputs, [0, 2, 3, 1], name='NCHW_to_NHWC') # input_shape = tf.shape(inputs) input_shape = inputs.get_shape() try: # tf pre-1.0 (top) vs 1.0 (bottom) output_shape = tf.pack([ int(input_shape[0]), 2 * int(input_shape[1]), 2 * int(input_shape[2]), output_dim ]) except Exception as e: output_shape = tf.stack([ int(input_shape[0]), 2 * int(input_shape[1]), 2 * int(input_shape[2]), output_dim ]) result = tf.nn.conv2d_transpose(value=inputs, filter=filters, output_shape=output_shape, strides=[1, 2, 2, 1], padding='SAME') if biases: _biases = lib.param(name + '.Biases', np.zeros(output_dim, dtype='float32')) result = tf.nn.bias_add(result, _biases) result = tf.transpose(result, [0, 3, 1, 2], name='NHWC_to_NCHW') return result
def Conv2D(name, input_dim, output_dim, filter_size, inputs, he_init=True, mask_type=None, stride=1, weightnorm=None, spectralnorm=False, update_collection = True, biases=True, gain=1.): """ inputs: tensor of shape (batch size, num channels, height, width) mask_type: one of None, 'a', 'b' returns: tensor of shape (batch size, num channels, height, width) """ def scope_has_variables(scope): return len(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope.name)) > 0 #with tf.name_scope(name) as scope: with tf.variable_scope(name) as scope: if scope_has_variables(scope): scope.reuse_variables() if mask_type is not None: mask_type, mask_n_channels = mask_type mask = np.ones( (filter_size, filter_size, input_dim, output_dim), dtype='float32' ) center = filter_size // 2 # Mask out future locations # filter shape is (height, width, input channels, output channels) mask[center+1:, :, :, :] = 0. mask[center, center+1:, :, :] = 0. # Mask out future channels for i in xrange(mask_n_channels): for j in xrange(mask_n_channels): if (mask_type=='a' and i >= j) or (mask_type=='b' and i > j): mask[ center, center, i::mask_n_channels, j::mask_n_channels ] = 0. def uniform(stdev, size): return np.random.uniform( low=-stdev * np.sqrt(3), high=stdev * np.sqrt(3), size=size ).astype('float32') fan_in = input_dim * filter_size**2 fan_out = output_dim * filter_size**2 / (stride**2) if mask_type is not None: # only approximately correct fan_in /= 2. fan_out /= 2. if he_init: filters_stdev = np.sqrt(4./(fan_in+fan_out)) else: # Normalized init (Glorot & Bengio) filters_stdev = np.sqrt(2./(fan_in+fan_out)) if _weights_stdev is not None: filter_values = uniform( _weights_stdev, (filter_size, filter_size, input_dim, output_dim) ) else: filter_values = uniform( filters_stdev, (filter_size, filter_size, input_dim, output_dim) ) # print "WARNING IGNORING GAIN" filter_values *= gain filters = lib.param(name+'.Filters', filter_values) if weightnorm==None: weightnorm = _default_weightnorm if weightnorm: norm_values = np.sqrt(np.sum(np.square(filter_values), axis=(0,1,2))) target_norms = lib.param( name + '.g', norm_values ) with tf.name_scope('weightnorm') as scope: norms = tf.sqrt(tf.reduce_sum(tf.square(filters), reduction_indices=[0,1,2])) filters = filters * (target_norms / norms) if spectralnorm: filters = spectral_normed_weight(W = filters, update_collection=update_collection) if mask_type is not None: with tf.name_scope('filter_mask'): filters = filters * mask result = tf.nn.conv2d( input=inputs, filter=filters, strides=[1, 1, stride, stride], padding='SAME', data_format='NCHW' ) if biases: _biases = lib.param( name+'.Biases', np.zeros(output_dim, dtype='float32') ) result = tf.nn.bias_add(result, _biases, data_format='NCHW') return result
def conv2d(inputs, num_outputs, kernel_size, sn, stride=1, rate=1, data_format='NCHW', activation_fn=tf.nn.relu, normalizer_fn=None, normalizer_params=None, weights_regularizer=None, weights_initializer=ly.xavier_initializer(), biases_initializer=init_ops.zeros_initializer(), biases_regularizer=None, reuse=None, scope=None): assert data_format == 'NCHW' assert rate == 1 if data_format == 'NCHW': channel_axis = 1 stride = [1, 1, stride, stride] rate = [1, 1, rate, rate] else: channel_axis = 3 stride = [1, stride, stride, 1] rate = [1, rate, rate, 1] input_dim = inputs.get_shape().as_list()[channel_axis] with tf.variable_scope(scope, 'Conv', [inputs], reuse=reuse) as sc: inputs = tf.convert_to_tensor(inputs) weights = tf.get_variable(name="weights", shape=(kernel_size, kernel_size, input_dim, num_outputs), initializer=weights_initializer, regularizer=weights_regularizer, trainable=True, dtype=inputs.dtype.base_dtype) # Spectral Normalization if sn: weights = spectral_normed_weight( weights, num_iters=1, update_collection=Config.SPECTRAL_NORM_UPDATE_OPS) conv_out = tf.nn.conv2d(inputs, weights, strides=stride, padding='SAME', data_format=data_format) if biases_initializer is not None: biases = tf.get_variable(name='biases', shape=(1, num_outputs, 1, 1), initializer=biases_initializer, regularizer=biases_regularizer, trainable=True, dtype=inputs.dtype.base_dtype) conv_out += biases if normalizer_fn is not None: normalizer_params = normalizer_params or {} conv_out = normalizer_fn(conv_out, activation_fn=None, **normalizer_params) if activation_fn is not None: conv_out = activation_fn(conv_out) return conv_out
def Linear(name, input_dim, output_dim, inputs, biases=True, initialization=None, weightnorm=None, spectralnorm=False, update_collection=None, gain=1.): """ initialization: None, `lecun`, 'glorot', `he`, 'glorot_he', `orthogonal`, `("uniform", range)` """ with tf.variable_scope(name) as scope: if scope_has_variables(scope): scope.reuse_variables() def uniform(stdev, size): if _weights_stdev is not None: stdev = _weights_stdev return np.random.uniform(low=-stdev * np.sqrt(3), high=stdev * np.sqrt(3), size=size).astype('float32') if initialization == 'lecun': # and input_dim != output_dim): # disabling orth. init for now because it's too slow weight_values = uniform(np.sqrt(1. / input_dim), (input_dim, output_dim)) elif initialization == 'glorot' or (initialization == None): weight_values = uniform(np.sqrt(2. / (input_dim + output_dim)), (input_dim, output_dim)) elif initialization == 'he': weight_values = uniform(np.sqrt(2. / input_dim), (input_dim, output_dim)) elif initialization == 'glorot_he': weight_values = uniform(np.sqrt(4. / (input_dim + output_dim)), (input_dim, output_dim)) elif initialization == 'orthogonal' or \ (initialization == None and input_dim == output_dim): # From lasagne def sample(shape): if len(shape) < 2: raise RuntimeError("Only shapes of length 2 or more are " "supported.") flat_shape = (shape[0], np.prod(shape[1:])) # TODO: why normal and not uniform? a = np.random.normal(0.0, 1.0, flat_shape) u, _, v = np.linalg.svd(a, full_matrices=False) # pick the one with the correct shape q = u if u.shape == flat_shape else v q = q.reshape(shape) return q.astype('float32') weight_values = sample((input_dim, output_dim)) elif initialization[0] == 'uniform': weight_values = np.random.uniform( low=-initialization[1], high=initialization[1], size=(input_dim, output_dim)).astype('float32') else: raise Exception('Invalid initialization!') weight_values *= gain weight = lib.param(name + '.W', weight_values) if weightnorm == None: weightnorm = _default_weightnorm if weightnorm: norm_values = np.sqrt(np.sum(np.square(weight_values), axis=0)) # norm_values = np.linalg.norm(weight_values, axis=0) target_norms = lib.param(name + '.g', norm_values) with tf.name_scope('weightnorm') as scope: norms = tf.sqrt( tf.reduce_sum(tf.square(weight), reduction_indices=[0])) weight = weight * (target_norms / norms) if spectralnorm: weight = spectral_normed_weight( weight, update_collection=update_collection) # if 'Discriminator' in name: # print "WARNING weight constraint on {}".format(name) # weight = tf.nn.softsign(10.*weight)*.1 if inputs.get_shape().ndims == 2: result = tf.matmul(inputs, weight) else: reshaped_inputs = tf.reshape(inputs, [-1, input_dim]) result = tf.matmul(reshaped_inputs, weight) result = tf.reshape( result, tf.pack(tf.unpack(tf.shape(inputs))[:-1] + [output_dim])) if biases: result = tf.nn.bias_add( result, lib.param(name + '.b', np.zeros((output_dim, ), dtype='float32'))) return result
def deconv2d(input_, output_shape, k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02, scale=1.0, with_learnable_sn_scale=False, with_sn=False, name="deconv2d", with_w=False, update_collection=None, data_format='NCHW', with_singular_values=False): with tf.variable_scope(name): scope_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, tf.get_variable_scope().name) has_summary = any([('w' in v.op.name) for v in scope_vars]) out_channel, in_channel = get_in_out_shape( output_shape, input_.get_shape().as_list(), data_format) strides = get_strides(d_h, d_w, data_format) # filter : [height, width, output_channels, in_channels] w = tf.get_variable( 'w', [k_h, k_w, out_channel, in_channel], initializer=tf.random_normal_initializer(stddev=stddev)) if with_sn: s = tf.get_variable('s', shape=[1], initializer=tf.constant_initializer(scale), trainable=with_learnable_sn_scale, dtype=tf.float32) w_bar, sigma = spectral_normed_weight( w, update_collection=update_collection, with_sigma=True) w_bar = s * w_bar deconv = tf.nn.conv2d_transpose(input_, w_bar, output_shape=output_shape, strides=strides, data_format=data_format) else: deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape, strides=strides, data_format=data_format) biases = tf.get_variable('biases', [out_channel], initializer=tf.constant_initializer(0.0)) deconv = tf.reshape( tf.nn.bias_add(deconv, biases, data_format=data_format), deconv.get_shape()) if not has_summary: if with_sn: variable_summaries({'b': biases, 's': s, 'sigma_w': sigma}) variable_summaries({'W': w}, with_singular_values=with_singular_values) else: variable_summaries({'b': biases}) variable_summaries({'W': w}, with_singular_values=with_singular_values) if with_w: return deconv, w, biases else: return deconv