def Net(aa, yt, x): s=aa.shape[1] with tf.sg_context(name='NNReg', stride=1, act='leaky_relu', bn=True, reuse=tf.AUTO_REUSE): yt=tf.expand_dims(yt,2) v1=tf.expand_dims(x,2).sg_conv(dim=16, size=(1,1), name='gen9',pad="SAME",bn=True) v2=v1.sg_conv(dim=64, size=(1,1), name='gen1',pad="SAME",bn=True) v3=v2.sg_conv(dim=128, size=(1,1), name='gen2',pad="SAME",bn=True) v4=v3.sg_conv(dim=256, size=(1,1), name='gen3',pad="SAME",bn=True) v5=v4.sg_conv(dim=512, size=(1,1), name='gen4',pad="SAME",bn=True) v5=tf.tile(tf.expand_dims(tf.reduce_max(v5, axis=1),axis=1),[1,s,1,1]) vv5=v5 v1=yt.sg_conv(dim=16, size=(1,1), name='gen99',pad="SAME",bn=True) v2=v1.sg_conv(dim=64, size=(1,1), name='gen11',pad="SAME",bn=True) v3=v2.sg_conv(dim=128, size=(1,1), name='gen22',pad="SAME",bn=True) v4=v3.sg_conv(dim=256, size=(1,1), name='gen33',pad="SAME",bn=True) v5=v4.sg_conv(dim=512, size=(1,1), name='gen44',pad="SAME",bn=True) v5=tf.tile(tf.expand_dims(tf.reduce_max(v5, axis=1),axis=1),[1,s,1,1]) ff=tf.concat([tf.expand_dims(aa,2),v5], axis=-1) ff=tf.concat([ff,vv5], axis=-1) f1=ff.sg_conv(dim=256, size=(1,1), name='f1',pad="SAME",bn=True) f2=f1.sg_conv(dim=128, size=(1,1), name='f2',pad="SAME",bn=True) f3=f2.sg_conv(dim=2, size=(1,1), name='f3',pad="SAME",bn=False, act="linear") f3=tf.squeeze(f3,axis=2) return f3
def sg_quasi_rnn(tensor, opt): # Split if opt.att: H, Z, F, O = tf.split(tensor, 4, axis=0) # (16, 150, 320) for all else: Z, F, O = tf.split(tensor, 3, axis=0) # (16, 150, 320) for all # step func def step(z, f, o, c): ''' Runs fo-pooling at each time step ''' c = f * c + (1 - f) * z if opt.att: # attention a = tf.nn.softmax(tf.einsum("ijk,ik->ij", H, c)) # alpha. (16, 150) k = (a.sg_expand_dims() * H).sg_sum( axis=1) # attentional sum. (16, 320) h = o * (k.sg_dense(act="linear") + \ c.sg_dense(act="linear")) else: h = o * c return h, c # hidden states, (new) cell memories # Do rnn loop c, hs = 0, [] timesteps = tensor.get_shape().as_list()[1] for t in range(timesteps): z = Z[:, t, :] # (16, 320) f = F[:, t, :] # (16, 320) o = O[:, t, :] # (16, 320) # apply step function h, c = step(z, f, o, c) # (16, 320), (16, 320) # save result hs.append(h.sg_expand_dims(axis=1)) # Concat to return H = tf.concat(hs, 1) # (16, 150, 320) seqlen = tf.to_int32( tf.reduce_sum(tf.sign(tf.abs(tf.reduce_sum(H, axis=-1))), 1)) # (16,) float32 h = tf.reverse_sequence(input=H, seq_lengths=seqlen, seq_dim=1)[:, 0, :] # last hidden state vector if opt.is_enc: H_z = tf.tile((h.sg_dense(act="linear").sg_expand_dims(axis=1)), [1, timesteps, 1]) H_f = tf.tile((h.sg_dense(act="linear").sg_expand_dims(axis=1)), [1, timesteps, 1]) H_o = tf.tile((h.sg_dense(act="linear").sg_expand_dims(axis=1)), [1, timesteps, 1]) concatenated = tf.concat([H, H_z, H_f, H_o], 0) # (16*4, 150, 320) return concatenated else: return H # (16, 150, 320)
def sg_quasi_rnn(tensor, opt): # Split if opt.att: H, Z, F, O = tf.split(axis=0, num_or_size_splits=4, value=tensor) # (16, 150, 320) for all else: Z, F, O = tf.split(axis=0, num_or_size_splits=3, value=tensor) # (16, 150, 320) for all # step func def step(z, f, o, c): ''' Runs fo-pooling at each time step ''' c = f * c + (1 - f) * z if opt.att: # attention a = tf.nn.softmax(tf.einsum("ijk,ik->ij", H, c)) # alpha. (16, 150) k = (a.sg_expand_dims() * H).sg_sum( dims=1) # attentional sum. (16, 150) h = o * (k.sg_dense(act="linear") + c.sg_dense(act="linear")) else: h = o * c return h, c # hidden states, (new) cell memories # Do rnn loop c, hs = 0, [] timesteps = tensor.get_shape().as_list()[1] for t in range(timesteps): z = Z[:, t, :] # (16, 320) f = F[:, t, :] # (16, 320) o = O[:, t, :] # (16, 320) # apply step function h, c = step(z, f, o, c) # (16, 320), (16, 320) # save result hs.append(h.sg_expand_dims(dim=1)) # Concat to return H = tf.concat(axis=1, values=hs) # (16, 150, 320) if opt.is_enc: H_z = tf.tile((h.sg_dense(act="linear").sg_expand_dims(dim=1)), [1, timesteps, 1]) H_f = tf.tile((h.sg_dense(act="linear").sg_expand_dims(dim=1)), [1, timesteps, 1]) H_o = tf.tile((h.sg_dense(act="linear").sg_expand_dims(dim=1)), [1, timesteps, 1]) concatenated = tf.concat(axis=0, values=[H, H_z, H_f, H_o]) # (16*4, 150, 320) return concatenated else: return H # (16, 150, 320)
def sg_quasi_conv1d(tensor, opt): opt += tf.sg_opt(is_enc=False) # Split into H and H_zfo H = tensor[:Hp.bs] H_z = tensor[Hp.bs:2 * Hp.bs] H_f = tensor[2 * Hp.bs:3 * Hp.bs] H_o = tensor[3 * Hp.bs:] if opt.is_enc: H_z, H_f, H_o = 0, 0, 0 # Convolution and merging with tf.sg_context(act="linear", causal=(not opt.is_enc), bn=opt.is_enc, ln=(not opt.is_enc)): Z = H.sg_aconv1d() + H_z # (16, 300, 320) F = H.sg_aconv1d() + H_f # (16, 300, 320) O = H.sg_aconv1d() + H_o # (16, 300, 320) # Activation Z = Z.sg_bypass(act="tanh") # (16, 300, 320) F = F.sg_bypass(act="sigmoid") # (16, 300, 320) O = O.sg_bypass(act="sigmoid") # (16, 300, 320) # Masking M = tf.sign(tf.abs(H))[:, :, :1] # (16, 300, 1) float32. 0 or 1 Z *= M # broadcasting F *= M # broadcasting O *= M # broadcasting # Concat ZFO = tf.concat(axis=0, values=[Z, F, O]) return ZFO # (16*3, 150, 320)
def sg_emb(**kwargs): r"""Returns a look-up table for embedding. kwargs: name: A name for the layer. emb: A 2-D array (optional). If None, the resulting tensor should have the shape of `[vocabulary size, embedding dimension size]`. Note that its first row is filled with 0's associated with padding. in_dim: A positive `integer`. The size of input dimension. dim: A positive `integer`. The size of output dimension. voca_size: A positive integer. The size of vocabulary. Returns: A 2-D `Tensor` of float32. """ opt = tf.sg_opt(kwargs) assert opt.name is not None, 'name is mandatory.' if opt.emb is None: # initialize embedding matrix assert opt.voca_size is not None, 'voca_size is mandatory.' assert opt.dim is not None, 'dim is mandatory.' w = tf.sg_initializer.he_uniform(opt.name, (opt.voca_size - 1, opt.dim)) else: # use given embedding matrix w = tf.sg_initializer.external(opt.name, value=opt.emb) # 1st row should be zero and not be updated by backprop because of zero padding. emb = tf.concat(0, [tf.zeros((1, opt.dim), dtype=tf.sg_floatx), w]) return emb
def sg_inverse_periodic_shuffle(tensor, opt): # default factor opt += tf.sg_opt(factor=2) # get current shape batch, row, col, channel = tensor.get_shape().as_list() # get target shape and channel num channel_factor = opt.factor * opt.factor # intermediate shape for shuffling shape_1 = [ batch, row / opt.factor, col / opt.factor, channel_factor // opt.factor, channel_factor // opt.factor ] shape_2 = [batch, row / opt.factor, col / opt.factor, channel_factor] # reshape and transpose for periodic shuffling for each channel out = [] for i in range(channel): out.append(tensor[:, :, :, i].sg_expand_dims().sg_reshape( shape=shape_1).sg_transpose(perm=(0, 1, 3, 2, 4)).sg_reshape(shape=shape_2)) # final output out = tf.concat(3, out) return tf.identity(out, name=opt.name)
def sg_emb(**kwargs): r"""Returns an embedding layer or a look-up table. Args: name: A name for the layer (required). emb: A 2-D array. Has the shape of `[vocabulary size -1, embedding dimension size]`. Note that the first row is filled with 0's because they correspond to padding. in_dim: A positive `integer`. The size of input dimension. dim: A positive `integer`. The size of output dimension. voca_size: A positive int32. Returns: A 2-D tensor. """ opt = tf.sg_opt(kwargs) assert opt.name is not None, 'name is mandatory.' import sg_initializer as init if opt.emb is None: # initialize embedding matrix assert opt.voca_size is not None, 'voca_size is mandatory.' assert opt.dim is not None, 'dim is mandatory.' w = init.he_uniform(opt.name, (opt.voca_size - 1, opt.dim)) else: # use given embedding matrix w = init.external(opt.name, value=opt.emb) # 1st row should be zero and not be updated by backprop because of zero padding. emb = tf.concat(0, [tf.zeros((1, opt.dim), dtype=tf.sg_floatx), w]) return emb
def rnn_classify(x, num_classes, is_test=False): with tf.sg_context(name='rnn_classify'): fw_cell = tf.nn.rnn_cell.MultiRNNCell( [lstm_cell(is_test) for _ in range(num_blocks)], state_is_tuple=True) bw_cell = tf.nn.rnn_cell.MultiRNNCell( [lstm_cell(is_test) for _ in range(num_blocks)], state_is_tuple=True) words_used_in_sent = tf.sign( tf.reduce_max(tf.abs(x), reduction_indices=2)) length = tf.cast( tf.reduce_sum(words_used_in_sent, reduction_indices=1), tf.int32) outputs, _ = tf.nn.bidirectional_dynamic_rnn(fw_cell, bw_cell, x, dtype=tf.float32, sequence_length=length) output = tf.concat(outputs, 2).sg_reshape(shape=[-1, 2 * latent_dim]) prediction = output.sg_dense(dim=num_classes, name='dense') res = tf.reshape(prediction, [x.get_shape().as_list()[0], -1, num_classes]) return res
def sg_periodic_shuffle(tensor, opt): # default factor opt += tf.sg_opt(factor=2) # get current shape batch, row, col, channel = tensor.get_shape().as_list() # get target channel num channel_target = channel / (opt.factor * opt.factor) channel_factor = channel / channel_target # intermediate shape for shuffling shape_1 = [ batch, row, col, channel_factor / opt.factor, channel_factor / opt.factor ] shape_2 = [batch, row * opt.factor, col * opt.factor, 1] # reshape and transpose for periodic shuffling for each channel out = [] for i in range(channel_target): out.append( (tensor[:, :, :, i * channel_factor:(i + 1) * channel_factor]).sg_reshape(shape=shape_1).sg_transpose( perm=(0, 1, 3, 2, 4)).sg_reshape(shape=shape_2)) # final output out = tf.concat(3, out) return tf.identity(out, name=opt.name)
def sg_rnn(tensor, opt): r"""Applies a simple rnn. Args: tensor: A 3-D `Tensor`. in_dim: A positive `integer`. The size of input dimension. dim: A positive `integer`. The size of output dimension. bias: Boolean. If True, biases are added. ln: Boolean. If True, layer normalization is applied. init_state: A 2-D `Tensor`. If None, the initial state is set to zeros. last_only: Boolean. If True, the outputs in the last time step are returned. Returns: A `Tensor`. If last_only is False, the output tensor has shape [batch size, time steps, dim]. If last_only is True, the shape will be [batch size, dim]. """ # layer normalization ln = lambda v: _ln_rnn(v, gamma, beta) if opt.ln else v # step function def step(h, x): # simple rnn ### Replace tensor[:, i, :] with x. bryan ### y = ln( tf.matmul(tensor[:, i, :], w) + tf.matmul(h, u) + (b if opt.bias else 0)) return y # parameter initialize w = init.orthogonal('W', (opt.in_dim, opt.dim)) u = init.identity('U', opt.dim) if opt.bias: b = init.constant('b', opt.dim) # layer normalization parameters if opt.ln: # offset, scale parameter beta = init.constant('beta', opt.dim) gamma = init.constant('gamma', opt.dim, value=1) # initial state init_h = opt.init_state if opt.init_state is not None \ else tf.zeros((tensor.get_shape().as_list()[0], opt.dim), dtype=tf.sg_floatx) # do rnn loop h, out = init_h, [] for i in range(tensor.get_shape().as_list()[1]): # apply step func h = step(h, tensor[:, i, :]) # save result out.append(h.sg_expand_dims(dim=1)) # merge tensor if opt.last_only: out = out[-1].sg_squeeze(dim=1) else: out = tf.concat(1, out) return out
def sg_densenet_layer(x, opt): r"""Applies basic architecture of densenet layer. Note that the fc layers in the original architecture will be replaced with fully convolutional layers. For convenience, We still call them fc layers, though. Args: x: A `Tensor`. opt: dim: An integer. Dimension for this resnet layer num: Number of times to repeat act: String. 'relu' (default). the activation function name trans: Boolean. If True(default), transition layer will be applied. reuse: Boolean(Optional). If True, all variables will be loaded from previous network. name: String. (optional) Used as convolution layer prefix Returns: A `Tensor`. """ assert opt.dim is not None, 'dim is mandatory.' assert opt.num is not None, 'num is mandatory.' # default stride opt += tf.sg_opt(stride=1, act='relu', trans=True) # format convolutional layer name def cname(index): return opt.name if opt.name is None else opt.name + '_%d' % index # dense layer with tf.sg_context(bias=False, reuse=opt.reuse): out = x for i in range(opt.num): # dense block out_new = (out.sg_bypass(act=opt.act, bn=True, name=cname(3 * i + 1)).sg_conv( dim=opt.dim // 4, size=1, act=opt.act, bn=True, name=cname(3 * i + 2)).sg_conv( dim=opt.dim, size=3, name=cname(3 * i + 3))) out = tf.concat([out_new, out], 3) # transition layer if opt.trans: out = (out.sg_bypass(act=opt.act, bn=True, name=cname(3 * i + 4)).sg_conv( size=1, name=cname(3 * i + 5)).sg_pool(avg=True)) return out
def q_process(t1, t2): ''' Processes each training sample so that it fits in the queue. ''' # Lstrip zeros zeros = tf.equal(t1, tf.zeros_like(t1)).sg_int().sg_sum() t1 = t1[zeros:] t2 = t2[zeros:] # zero-PrePadding t1 = tf.concat([tf.zeros([Hyperparams.seqlen-1], tf.int32), t1], 0)# 49 zero-prepadding t2 = tf.concat([tf.zeros([Hyperparams.seqlen-1], tf.int32), t2], 0)# 49 zero-prepadding # radom crop stacked = tf.stack((t1, t2)) cropped = tf.random_crop(stacked, [2, Hyperparams.seqlen]) t1, t2 = cropped[0], cropped[1] t2 = t2[-1] return t1, t2
def rnn_classify(x, num_classes, is_test=False): with tf.sg_context(name='rnn_classify'): fw_cell = tf.nn.rnn_cell.MultiRNNCell([lstm_cell(is_test) for _ in range(num_blocks)], state_is_tuple=True) bw_cell = tf.nn.rnn_cell.MultiRNNCell([lstm_cell(is_test) for _ in range(num_blocks)], state_is_tuple=True) words_used_in_sent = tf.sign(tf.reduce_max(tf.abs(x), reduction_indices=2)) length = tf.cast(tf.reduce_sum(words_used_in_sent, reduction_indices=1), tf.int32) outputs, _ = tf.nn.bidirectional_dynamic_rnn(fw_cell, bw_cell, x, dtype=tf.float32, sequence_length=length) output = tf.concat(outputs, 2).sg_reshape(shape=[-1, 2 * latent_dim]) prediction = output.sg_dense(dim=num_classes, name='dense') res = tf.reshape(prediction, [x.get_shape().as_list()[0], -1, num_classes]) return res
def sg_rnn(tensor, opt): # layer normalization ln = lambda v: _ln_rnn(v, gamma, beta) if opt.ln else v # step function def step(h, x): # simple rnn y = ln( tf.matmul(tensor[:, i, :], w) + tf.matmul(h, u) + (b if opt.bias else 0)) return y # parameter initialize w = init.orthogonal('W', (opt.in_dim, opt.dim)) u = init.identity('U', opt.dim) if opt.bias: b = init.constant('b', opt.dim) # layer normalization parameters if opt.ln: # offset, scale parameter beta = init.constant('beta', opt.dim) gamma = init.constant('gamma', opt.dim, value=1) # initial state init_h = opt.init_state if opt.init_state is not None \ else tf.zeros((tensor.get_shape().as_list()[0], opt.dim), dtype=tf.sg_floatx) # do rnn loop h, out = init_h, [] for i in range(tensor.get_shape().as_list()[1]): # apply step func h = step(h, tensor[:, i, :]) # save result out.append(h.sg_expand_dims(dim=1)) # merge tensor if opt.last_only: out = out[-1].sg_squeeze(dim=1) else: out = tf.concat(1, out) return out
def sg_emb(**kwargs): opt = tf.sg_opt(kwargs) assert opt.name is not None, 'name is mandatory.' import sg_initializer as init if opt.emb is None: # initialize embedding matrix assert opt.voca_size is not None, 'voca_size is mandatory.' assert opt.dim is not None, 'dim is mandatory.' w = init.he_uniform(opt.name, (opt.voca_size - 1, opt.dim)) else: # use given embedding matrix w = init.external(opt.name, value=opt.emb) # 1st row should be zero and not be updated by backprop because of zero padding. emb = tf.concat(0, [tf.zeros((1, opt.dim), dtype=tf.sg_floatx), w]) return emb
def sg_periodic_shuffle(tensor, opt): r""" Periodic shuffle transformation for SubPixel CNN. (see [Shi et al. 2016](http://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Shi_Real-Time_Single_Image_CVPR_2016_paper.pdf) Args: tensor: A tensor (automatically given by chain). opt: factor: factor to multiply shape by. Default is 2. name : If provided, it replaces current tensor's name. Returns: A tensor """ # default factor opt += tf.sg_opt(factor=2) # get current shape batch, row, col, channel = tensor.get_shape().as_list() # get target channel num channel_target = channel // (opt.factor * opt.factor) channel_factor = channel // channel_target # intermediate shape for shuffling shape_1 = [ batch, row, col, channel_factor // opt.factor, channel_factor // opt.factor ] shape_2 = [batch, row * opt.factor, col * opt.factor, 1] # reshape and transpose for periodic shuffling for each channel out = [] for i in range(channel_target): out.append( (tensor[:, :, :, i * channel_factor:(i + 1) * channel_factor]).sg_reshape(shape=shape_1).sg_transpose( perm=(0, 1, 3, 2, 4)).sg_reshape(shape=shape_2)) # final output out = tf.concat(axis=3, values=out) return tf.identity(out, name=opt.name)
def sg_concat(tensor, opt): r"""Concatenates tensors along a axis. See `tf.concat()` in tensorflow. Args: tensor: A `Tensor` (automatically given by chain). opt: target: A `Tensor`. Must have the same rank as `tensor`, and all dimensions except `opt.dim` must be equal. axis : Target axis. Default is the last one. name: If provided, replace current tensor's name. Returns: A `Tensor`. """ assert opt.target is not None, 'target is mandatory.' opt += tf.sg_opt(axis=tensor.get_shape().ndims-1) target = opt.target if isinstance(opt.target, (tuple, list)) else [opt.target] return tf.concat([tensor] + target, opt.axis, name=opt.name)
def sg_quasi_conv1d(tensor, opt): ''' Args: tensor: A 3-D tensor of either [batch size, time steps, embedding size] for original X or [batch size * 4, time steps, embedding size] for the others. ''' opt += tf.sg_opt(is_enc=False) # Split into H and H_zfo H = tensor[:Hp.batch_size] H_z = tensor[Hp.batch_size:2 * Hp.batch_size] H_f = tensor[2 * Hp.batch_size:3 * Hp.batch_size] H_o = tensor[3 * Hp.batch_size:] if opt.is_enc: H_z, H_f, H_o = 0, 0, 0 # Convolution and merging with tf.sg_context(size=opt.size, act="linear", causal=(not opt.is_enc)): Z = H.sg_aconv1d() + H_z # (16, 150, 320) F = H.sg_aconv1d() + H_f # (16, 150, 320) O = H.sg_aconv1d() + H_o # (16, 150, 320) # Activation with tf.sg_context(ln=True): Z = Z.sg_bypass(act="tanh") # (16, 150, 320) F = F.sg_bypass(act="sigmoid") # (16, 150, 320) O = O.sg_bypass(act="sigmoid") # (16, 150, 320) # Masking M = tf.sign(tf.abs(tf.reduce_sum( H, axis=-1, keep_dims=True))) # (16, 150, 1) float32. 0 or 1 Z *= M # broadcasting F *= M # broadcasting O *= M # broadcasting # Concat ZFO = tf.concat([Z, F, O], 0) return ZFO # (16*3, 150, 320)
def sg_quasi_conv1d(tensor, opt): opt += tf.sg_opt(is_enc=False, causal=True) # Split into H and H_zfo H = tensor[:Hp.batch_size] H_z = tensor[Hp.batch_size:2 * Hp.batch_size] H_f = tensor[2 * Hp.batch_size:3 * Hp.batch_size] H_o = tensor[3 * Hp.batch_size:] if opt.is_enc: H_z, H_f, H_o = 0, 0, 0 # Convolution and merging with tf.sg_context(size=opt.size, act="linear", causal=opt.causal and (not opt.is_enc), dev=opt.dev, reuse=opt.reuse_vars): Z = H.sg_aconv1d_gpus(name="aconvz_" + opt.name) + H_z # (b, seqlen, hd) F = H.sg_aconv1d_gpus(name="aconvf_" + opt.name) + H_f # (b, seqlen, hd) O = H.sg_aconv1d_gpus(name="aconvo_" + opt.name) + H_o # (b, seqlen, hd) # Activation with tf.sg_context(dev=opt.dev, reuse=opt.reuse_vars): Z = Z.sg_bypass_gpus(act="tanh", name="tanhz_" + opt.name) # (b, seqlen, hd) F = F.sg_bypass_gpus(act="sigmoid", name="sigmf_" + opt.name) # (b, seqlen, hd) O = O.sg_bypass_gpus(act="sigmoid", name="sigmo_" + opt.name) # (b, seqlen, hd) ZFO = tf.concat([Z, F, O], 0) return ZFO # (batch*3, seqlen, hiddim)
# inputs # # MNIST input tensor ( with QueueRunner ) data = tf.sg_data.Mnist(batch_size=batch_size) # input images and label x = data.train.image y = data.train.label # labels for discriminator y_real = tf.ones(batch_size) y_fake = tf.zeros(batch_size) # discriminator labels ( half 1s, half 0s ) y_disc = tf.concat(0, [y, y * 0]) # categorical latent variable z_cat = tf.multinomial( tf.ones((batch_size, cat_dim), dtype=tf.sg_floatx) / cat_dim, 1).sg_squeeze().sg_int() # continuous latent variable z_con = tf.random_normal((batch_size, con_dim)) # random latent variable dimension z_rand = tf.random_normal((batch_size, rand_dim)) # latent variable z = tf.concat(1, [z_cat.sg_one_hot(depth=cat_dim), z_con, z_rand]) # # Computational graph #
# hyper parameters # batch_size = 16 # batch size # # inputs # # ComTrans parallel corpus input tensor ( with QueueRunner ) data = ComTrans(batch_size=batch_size) # source, target sentence x, y = data.source, data.target # shift target for training source y_in = tf.concat([tf.zeros((batch_size, 1), tf.sg_intx), y[:, :-1]], axis=1) # vocabulary size voca_size = data.voca_size # make embedding matrix for source and target emb_x = tf.sg_emb(name='emb_x', voca_size=voca_size, dim=latent_dim) emb_y = tf.sg_emb(name='emb_y', voca_size=voca_size, dim=latent_dim) # latent from embed table z_x = x.sg_lookup(emb=emb_x) z_y = y_in.sg_lookup(emb=emb_y) # encode graph ( atrous convolution ) enc = encode(z_x) # concat merge target source
# inputs # # MNIST input tensor ( with QueueRunner ) data = tf.sg_data.Mnist(batch_size=batch_size) # input images and label x = data.train.image y = data.train.label # labels for discriminator y_real = tf.ones(batch_size) y_fake = tf.zeros(batch_size) # discriminator labels ( half 1s, half 0s ) y_disc = tf.concat([y, y * 0], 0) # categorical latent variable z_cat = tf.multinomial( tf.ones((batch_size, cat_dim), dtype=tf.sg_floatx) / cat_dim, 1).sg_squeeze().sg_int() # continuous latent variable z_con = tf.random_normal((batch_size, con_dim)) # random latent variable dimension z_rand = tf.random_normal((batch_size, rand_dim)) # latent variable z = tf.concat(axis=1, values=[z_cat.sg_one_hot(depth=cat_dim), z_con, z_rand]) # # Computational graph #
# MNIST input tensor ( with QueueRunner ) data = tf.sg_data.Mnist(batch_size=batch_size) # input images x = data.train.image # corrupted image x_small = tf.image.resize_bicubic(x, (14, 14)) x_nearest = tf.image.resize_images(x_small, (28, 28), tf.image.ResizeMethod.NEAREST_NEIGHBOR) # generator labels ( all ones ) y = tf.ones(batch_size, dtype=tf.sg_floatx) # discriminator labels ( half 1s, half 0s ) y_disc = tf.concat([y, y * 0], 0) # # create generator # # I've used ESPCN scheme # http://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Shi_Real-Time_Single_Image_CVPR_2016_paper.pdf # # generator network with tf.sg_context(name='generator', act='relu', bn=True): gen = (x_small.sg_conv(dim=32).sg_conv().sg_conv( dim=4, act='sigmoid', bn=False).sg_periodic_shuffle(factor=2)) # add image summary tf.sg_summary_image(gen)
num_cont = 2 # continuous variable number num_dim = 30 # total latent dimension ( category + continuous + noise ) # # inputs # # input tensor ( with QueueRunner ) data = TimeSeriesData(batch_size=batch_size) x = data.X # generator labels ( all ones ) y = tf.ones(batch_size, dtype=tf.sg_floatx) # discriminator labels ( half 1s, half 0s ) y_disc = tf.concat(0, [y, y * 0]) # # create generator # # random class number z_cat = tf.multinomial(tf.ones((batch_size, num_category), dtype=tf.sg_floatx) / num_category, 1).sg_squeeze() # random seed = random categorical variable + random uniform z = z_cat.sg_one_hot(depth=num_category).sg_concat(target=tf.random_uniform((batch_size, num_dim-num_category))) # random continuous variable z_cont = z[:, num_category:num_category+num_cont]
x = data.train.image # labels for discriminator y_real = tf.ones(batch_size) y_fake = tf.zeros(batch_size) # categorical latent variable z_cat = tf.multinomial( tf.ones((batch_size, cat_dim), dtype=tf.sg_floatx) / cat_dim, 1).sg_squeeze().sg_int() # continuous latent variable z_con = tf.random_uniform((batch_size, con_dim)) # random latent variable dimension z_rand = tf.random_uniform((batch_size, rand_dim)) # latent variable z = tf.concat([z_cat.sg_one_hot(depth=cat_dim), z_con, z_rand], 1) # # Computational graph # # generator gen = generator(z) # add image summary tf.sg_summary_image(x, name='real') tf.sg_summary_image(gen, name='fake') # discriminator disc_real, _, _ = discriminator(x) disc_fake, cat_fake, con_fake = discriminator(gen)
# MNIST input tensor ( with QueueRunner ) data = tf.sg_data.Mnist(batch_size=batch_size) # input images x = data.train.image # corrupted image x_small = tf.image.resize_bicubic(x, (14, 14)) x_nearest = tf.image.resize_images(x_small, (28, 28), tf.image.ResizeMethod.NEAREST_NEIGHBOR) # generator labels ( all ones ) y = tf.ones(batch_size, dtype=tf.sg_floatx) # discriminator labels ( half 1s, half 0s ) y_disc = tf.concat(0, [y, y * 0]) # # create generator # # I've used ESPCN scheme # http://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Shi_Real-Time_Single_Image_CVPR_2016_paper.pdf # # generator network with tf.sg_context(name='generator', act='relu', bn=True): gen = (x_small .sg_conv(dim=32) .sg_conv() .sg_conv(dim=4, act='sigmoid', bn=False) .sg_periodic_shuffle(factor=2))
def sg_concat(tensor, opt): assert opt.target is not None, 'target is mandatory.' opt += tf.sg_opt(dim=tensor.get_shape().ndims - 1) target = opt.target if isinstance(opt.target, (tuple, list)) else [opt.target] return tf.concat(opt.dim, [tensor] + target, name=opt.name)
# convert to tensor queue self.train.image, self.train.label = \ _data_to_tensor([x.astype('float32'), label.astype('int32')], batch_size, name='train') # calc total batch count self.train.num_batch = label.shape[0] // batch_size # MNIST input tensor ( with QueueRunner ) data = Mnist(batch_size=batch_size) # input images x = data.train.image # generator labels ( all ones ) y = tf.ones(batch_size, dtype=tf.sg_floatx) # discriminator labels ( half 1s, half 0s ) y_disc = tf.concat([y, y * 0], 0) # # create generator # # random class number z_cat = tf.multinomial( tf.ones((batch_size, num_category), dtype=tf.sg_floatx) / num_category, 1).sg_squeeze().sg_int() # random seed = random categorical variable + random uniform z = z_cat.sg_one_hot(depth=num_category).sg_concat( target=tf.random_uniform((batch_size, num_dim - num_category))) # random continuous variable z_cont = z[:, num_category:num_category + num_cont] # category label label = tf.concat([data.train.label, z_cat], 0) # generator network
num_cont = 2 # continuous variable number num_dim = 30 # total latent dimension ( category + continuous + noise ) # # inputs # # input tensor ( with QueueRunner ) data = TimeSeriesData(batch_size=batch_size) x = data.X # generator labels ( all ones ) y = tf.ones(batch_size, dtype=tf.sg_floatx) # discriminator labels ( half 1s, half 0s ) y_disc = tf.concat(0, [y, y * 0]) # # create generator # # random class number z_cat = tf.multinomial( tf.ones((batch_size, num_category), dtype=tf.sg_floatx) / num_category, 1).sg_squeeze() # random seed = random categorical variable + random uniform z = z_cat.sg_one_hot(depth=num_category).sg_concat( target=tf.random_uniform((batch_size, num_dim - num_category))) # random continuous variable
def sg_lstm(tensor, opt): r"""Applies an LSTM. Args: tensor: A 3-D `Tensor`. in_dim: A positive `integer`. The size of input dimension. dim: A positive `integer`. The size of output dimension. bias: Boolean. If True, biases are added. ln: Boolean. If True, layer normalization is applied. init_state: A 2-D `Tensor`. If None, the initial state is set to zeros. last_only: Boolean. If True, the outputs in the last time step are returned. Returns: A `Tensor`. If last_only is False, the output tensor has shape [batch size, time steps, dim]. If last_only is True, the shape will be [batch size, dim]. """ # layer normalization ln = lambda v: _ln_rnn(v, gamma, beta) if opt.ln else v # step func def step(h, c, x): # forget gate f = tf.sigmoid( ln( tf.matmul(x, w_f) + tf.matmul(h, u_f) + (b_f if opt.bias else 0))) # input gate i = tf.sigmoid( ln( tf.matmul(x, w_i) + tf.matmul(h, u_i) + (b_i if opt.bias else 0))) # new cell value cc = tf.tanh( ln( tf.matmul(x, w_c) + tf.matmul(h, u_c) + (b_c if opt.bias else 0))) # out gate o = tf.sigmoid( ln( tf.matmul(x, w_o) + tf.matmul(h, u_o) + (b_o if opt.bias else 0))) # cell update cell = f * c + i * cc # final output y = o * tf.tanh(cell) return y, cell # parameter initialize w_i = init.orthogonal('W_i', (opt.in_dim, opt.dim)) u_i = init.identity('U_i', opt.dim) w_f = init.orthogonal('W_f', (opt.in_dim, opt.dim)) u_f = init.identity('U_f', opt.dim) w_o = init.orthogonal('W_o', (opt.in_dim, opt.dim)) u_o = init.identity('U_o', opt.dim) w_c = init.orthogonal('W_c', (opt.in_dim, opt.dim)) u_c = init.identity('U_c', opt.dim) if opt.bias: b_i = init.constant('b_i', opt.dim) b_f = init.constant('b_f', opt.dim) b_o = init.constant('b_o', opt.dim, value=1) b_c = init.constant('b_c', opt.dim) # layer normalization parameters if opt.ln: # offset, scale parameter beta = init.constant('beta', opt.dim) gamma = init.constant('gamma', opt.dim, value=1) # initial state init_h = opt.init_state if opt.init_state is not None \ else tf.zeros((tensor.get_shape().as_list()[0], opt.dim), dtype=tf.sg_floatx) # do rnn loop h, c, out = init_h, init_h, [] for i in range(tensor.get_shape().as_list()[1]): # apply step function h, c = step(h, c, tensor[:, i, :]) # save result out.append(h.sg_expand_dims(dim=1)) # merge tensor if opt.last_only: out = out[-1].sg_squeeze(dim=1) else: out = tf.concat(1, out) return out
def sg_gru(tensor, opt): r"""Applies a GRU. Args: tensor: A 3-D `Tensor`. in_dim: A positive `integer`. The size of input dimension. dim: A positive `integer`. The size of output dimension. bias: Boolean. If True, biases are added. ln: Boolean. If True, layer normalization is applied. init_state: A 2-D `Tensor`. If None, the initial state is set to zeros. last_only: Boolean. If True, the outputs in the last time step are returned. Returns: A `Tensor`. If last_only is False, the output tensor has shape [batch size, time steps, dim]. If last_only is True, the shape will be [batch size, dim]. """ # layer normalization ln = lambda v: _ln_rnn(v, gamma, beta) if opt.ln else v # step func def step(h, x): # update gate z = tf.sigmoid( ln( tf.matmul(x, w_z) + tf.matmul(h, u_z) + (b_z if opt.bias else 0))) # reset gate r = tf.sigmoid( ln( tf.matmul(x, w_r) + tf.matmul(h, u_r) + (b_r if opt.bias else 0))) # h_hat hh = tf.tanh( ln( tf.matmul(x, w_h) + tf.matmul(r * h, u_h) + (b_h if opt.bias else 0))) # final output y = (1. - z) * h + z * hh return y # parameter initialize w_z = init.orthogonal('W_z', (opt.in_dim, opt.dim)) u_z = init.identity('U_z', opt.dim) w_r = init.orthogonal('W_r', (opt.in_dim, opt.dim)) u_r = init.identity('U_r', opt.dim) w_h = init.orthogonal('W_h', (opt.in_dim, opt.dim)) u_h = init.identity('U_h', opt.dim) if opt.bias: b_z = init.constant('b_z', opt.dim) b_r = init.constant('b_r', opt.dim) b_h = init.constant('b_h', opt.dim) # layer normalization parameters if opt.ln: # offset, scale parameter beta = init.constant('beta', opt.dim) gamma = init.constant('gamma', opt.dim, value=1) # initial state init_h = opt.init_state if opt.init_state is not None \ else tf.zeros((tensor.get_shape().as_list()[0], opt.dim), dtype=tf.sg_floatx) # do rnn loop h, out = init_h, [] for i in range(tensor.get_shape().as_list()[1]): # apply step function h = step(h, tensor[:, i, :]) # save result out.append(h.sg_expand_dims(dim=1)) # merge tensor if opt.last_only: out = out[-1].sg_squeeze(dim=1) else: out = tf.concat(1, out) return out
def sg_quasi_rnn(tensor, opt): # Split if opt.att: H, Z, F, O = tf.split(axis=0, num_or_size_splits=4, value=tensor) # (b, seqlen, hd) for all else: Z, F, O = tf.split(axis=0, num_or_size_splits=3, value=tensor) # (b, seqlen, hd) for all # step func def step(z, f, o, c): ''' Runs fo-pooling at each time step ''' c = f * c + (1 - f) * z if opt.att: # attention a = tf.nn.softmax(tf.einsum("ijk,ik->ij", H, c)) # alpha. (b, seqlen) k = (a.sg_expand_dims() * H).sg_sum( axis=1) # attentional sum. (b, seqlen) h = o * (k.sg_dense_gpus(act="linear",name = "k%d_%s"%(t,opt.name),dev = opt.dev,reuse=opt.reuse_vars)\ + c.sg_dense_gpus(act="linear",name = "c%d_%s"%(t,opt.name),dev = opt.dev,reuse=opt.reuse_vars)) else: h = o * c return h, c # hidden states, (new) cell memories # Do rnn loop c, hs = 0, [] timesteps = tensor.get_shape().as_list()[1] for t in range(timesteps): z = Z[:, t, :] # (b, hd) f = F[:, t, :] # (b, hd) o = O[:, t, :] # (b, hd) # apply step function h, c = step(z, f, o, c) # (b, hd), (b, hd) # save result hs.append(h.sg_expand_dims(axis=1)) # Concat to return H = tf.concat(hs, 1) # (b, seqlen, hd) if opt.is_enc: H_z = tf.tile( (h.sg_dense_gpus(act="linear", name="z_%s" % (opt.name), dev=opt.dev, reuse=opt.reuse_vars).sg_expand_dims(axis=1)), [1, timesteps, 1]) H_f = tf.tile( (h.sg_dense_gpus(act="linear", name="f_%s" % (opt.name), dev=opt.dev, reuse=opt.reuse_vars).sg_expand_dims(axis=1)), [1, timesteps, 1]) H_o = tf.tile( (h.sg_dense_gpus(act="linear", name="o_%s" % (opt.name), dev=opt.dev, reuse=opt.reuse_vars).sg_expand_dims(axis=1)), [1, timesteps, 1]) concatenated = tf.concat(axis=0, values=[H, H_z, H_f, H_o]) # (b*4, seqlen, hd) return concatenated else: return H # (b, seqlen, hd)
# # input tensor ( with QueueRunner ) data = SpectrumData(batch_size=batch_size) x = data.X # labels for discriminator y_real = tf.ones(batch_size) y_fake = tf.zeros(batch_size) # continuous latent variable z_con = tf.random_uniform((batch_size, con_dim)) # random latent variable dimension z_rand = tf.random_uniform((batch_size, rand_dim)) # latent variable z = tf.concat([z_con, z_rand], 1) # # Computational graph # # generator gen = generator(z) # discriminator disc_real, _ = discriminator(x) disc_fake, con_fake = discriminator(gen) # # loss #