def encode_layer(self, input_, dilation, layer_no):
options = self.options
relu1 = tf.nn.relu(input_, name = 'enc_relu1_layer{}'.format(layer_no))
conv1 = ops.conv1d(relu1, options['residual_channels'], name = 'enc_conv1d_1_layer{}'.format(layer_no))
relu2 = tf.nn.relu(conv1, name = 'enc_relu2_layer{}'.format(layer_no))
dilated_conv = ops.dilated_conv1d(relu2, options['residual_channels'],
dilation, options['encoder_filter_width'],
causal = False,
name = "enc_dilated_conv_layer{}".format(layer_no)
)
relu3 = tf.nn.relu(dilated_conv, name = 'enc_relu1_layer{}'.format(layer_no))
conv2 = ops.conv1d(relu3, 2 * options['residual_channels'], name = 'enc_conv1d_2_layer{}'.format(layer_no))
return input_ + conv2
def build_generator(self, reuse=False):
if reuse:
tf.get_variable_scope().reuse_variables()
options = self.options
self.seed_sentence = tf.placeholder('int32', [None, None],
name='seed_sentence')
source_embedding = tf.nn.embedding_lookup(self.w_sentence_embedding,
self.seed_sentence,
name="source_embedding")
curr_input = source_embedding
for layer_no, dilation in enumerate(options['dilations']):
curr_input = ops.byetenet_residual_block(
curr_input,
dilation,
layer_no,
options['residual_channels'],
options['filter_width'],
causal=True,
train=False)
logits = ops.conv1d(tf.nn.relu(curr_input),
options['vocab_size'],
name='logits')
logits_flat = tf.reshape(logits, [-1, options['vocab_size']])
probs_flat = tf.nn.softmax(logits_flat)
self.g_probs = tf.reshape(
probs_flat,
[-1, tf.shape(self.seed_sentence)[1], options['vocab_size']])
def generator(self, state_logits, last_idx, vocab_dim, reuse=False):
with tf.variable_scope("gen", reuse=reuse):
#h: (mb_size, seq_len, 128)
h = ops.conv1d(state_logits, 128, dilation=1, k_w=1, name="conv1")
h = tf.nn.relu(h)
#logits: (mb_size, seq_len, vocab_dim)
#logits_last: (mb_size, vocab_dim)
#greedy_pred: (mb_size, seq_len)
#prob: (mb_size, seq_len, vocab_dim)
logits = ops.conv1d(h, vocab_dim, dilation=1, k_w=1, name="conv_logits")
logits_last = tf.gather_nd(logits, last_idx)
greedy_pred = tf.argmax(logits, axis=-1)
prob = tf.nn.softmax(logits, axis=-1)
return logits, logits_last, greedy_pred, prob
def predict_graph(self, is_negsample=False, reuse=False):
if reuse:
tf.get_variable_scope().reuse_variables()
self.input_predict = tf.placeholder('int32', [None, None],
name='input_predict')
label_seq, dilate_input = self.model_graph(self.input_predict,
train=False)
model_para = self.model_para
if is_negsample:
logits_2D = tf.reshape(dilate_input[:, -1:, :],
[-1, model_para['dilated_channels']])
logits_2D = tf.matmul(logits_2D, tf.transpose(self.softmax_w))
logits_2D = tf.nn.bias_add(logits_2D, self.softmax_b)
else:
logits = ops.conv1d(tf.nn.relu(dilate_input[:, -1:, :]),
model_para['item_size'],
name='logits')
logits_2D = tf.reshape(logits, [-1, model_para['item_size']])
label_flat = tf.reshape(label_seq[:, -1], [-1])
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=label_flat, logits=logits_2D)
self.loss_test = tf.reduce_mean(loss)
probs_flat = tf.nn.softmax(logits_2D)
self.g_probs = tf.reshape(probs_flat,
[-1, 1, model_para['item_size']]) #slow
self.top_k = tf.nn.top_k(
self.g_probs[:, -1], k=5, name='top-k'
) # much faster, if you change [probs] = sess.run([itemrec.g_probs],..)to [top_k] = sess.run([itemrec.top_k],..) in NextitNet_TF_Pretrain.py
def predict_graph(self, is_negsample=False, reuse=False):
if reuse:
tf.get_variable_scope().reuse_variables()
self.input_predict = tf.placeholder('int32', [None, None],
name='input_predict')
label_seq, dilate_input = self.model_graph(self.input_predict,
train=False)
model_para = self.model_para
if is_negsample:
logits_2D = tf.reshape(dilate_input[:, -1:, :],
[-1, model_para['dilated_channels']])
logits_2D = tf.matmul(logits_2D, tf.transpose(self.softmax_w))
logits_2D = tf.nn.bias_add(logits_2D, self.softmax_b)
else:
logits = ops.conv1d(tf.nn.relu(dilate_input[:, -1:, :]),
model_para['item_size'],
name='logits')
logits_2D = tf.reshape(logits, [-1, model_para['item_size']])
label_flat = tf.reshape(label_seq[:, -1], [-1])
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=label_flat, logits=logits_2D)
self.loss_test = tf.reduce_mean(loss)
probs_flat = tf.nn.softmax(logits_2D)
self.g_probs = tf.reshape(probs_flat, [-1, 1, model_para['item_size']])
def predict_graph_onrecall(self, is_negsample=False, reuse=False):
if reuse:
tf.get_variable_scope().reuse_variables()
self.input_predict = tf.placeholder('int32', [None, None], name='input_predict')
self.input_recall = tf.placeholder('int32', [None, None], name='input_recall')# candidate items
label_seq, dilate_input = self.model_graph(self.input_predict, train=False)
model_para = self.model_para
if is_negsample:
logits_2D=dilate_input[:, -1:, :]
recall_mat = tf.nn.embedding_lookup(self.softmax_w, self.input_recall)
logits_2D = tf.matmul(logits_2D, tf.transpose(recall_mat,[0,2,1]))
logits_2D=tf.reshape(logits_2D, [-1, tf.shape(self.input_recall)[1]])
recall_bias = tf.nn.embedding_lookup(self.softmax_b, self.input_recall)
logits_2D=tf.add(logits_2D,recall_bias)
else:
# logits = ops.conv1d(tf.nn.relu(dilate_input), model_para['item_size'], name='logits')
logits = ops.conv1d(tf.nn.relu(dilate_input[:, -1:, :]), model_para['item_size'], name='logits')
logits_2D = tf.reshape(logits, [-1, model_para['item_size']])
probs_flat = tf.nn.softmax(logits_2D, name='softmax')
self.g_probs = probs_flat
def get_masked_lm_output(self, bert_config, input_tensor, positions,
label_ids, label_weights, trainable=True):
"""Get loss and log probs for the masked LM."""
input_tensor = self.gather_indexes(input_tensor, positions)
if self.is_negsample:
logits_2D = input_tensor
label_flat = tf.reshape(label_ids, [-1, 1]) # 1 is the number of positive example
num_sampled = int(0.2 * self.model_para['item_size']) # sample 20% as negatives
loss = tf.nn.sampled_softmax_loss(self.softmax_w, self.softmax_b, label_flat, logits_2D,
num_sampled,
self.model_para['item_size'])
else:
sequence_shape = modeling.get_shape_list(positions)
batch_size = sequence_shape[0]
seq_length = sequence_shape[1]
residual_channels = input_tensor.get_shape().as_list()[-1]
input_tensor = tf.reshape(input_tensor, [-1, seq_length, residual_channels])
logits = ops.conv1d(tf.nn.relu(input_tensor), self.model_para['item_size'], name='logits')
logits_2D = tf.reshape(logits, [-1, self.model_para['item_size']])
label_flat = tf.reshape(label_ids, [-1])
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=label_flat, logits=logits_2D)
loss = tf.reduce_mean(loss)
# not sure the impact, 0.001 is empirical value, for large session data it is not very userful
regularization = 0.001 * tf.reduce_mean([tf.nn.l2_loss(v) for v in tf.trainable_variables()])
loss=loss+regularization
return loss
def attend_image(self,
image_features,
encoded_question,
dropout_keep_prob=1.0):
options = self.options
img_dim = options['img_dim']
img_channels = options['img_channels']
with tf.variable_scope("Attention"):
encoded_question_exp = tf.expand_dims(encoded_question, dim=1)
encoded_question_tiled = tf.tile(encoded_question_exp,
[1, img_dim * img_dim, 1])
image_features_flat = tf.reshape(
image_features, [-1, img_dim * img_dim, img_channels])
combined_features = tf.concat(
[image_features_flat, encoded_question_tiled], axis=2)
combined_features = tf.nn.dropout(combined_features,
dropout_keep_prob)
combined_features = ops.conv1d(combined_features,
512,
name="conv_1")
combined_features = tf.nn.relu(combined_features)
combined_features = tf.nn.dropout(combined_features,
dropout_keep_prob)
logits = ops.conv1d(combined_features, 2, name="conv_2")
prob1 = tf.nn.softmax(logits[:, :, 0], name="prob_map_1")
prob2 = tf.nn.softmax(logits[:, :, 1], name="prob_map_2")
glimplse1 = tf.reduce_sum(
image_features_flat * tf.expand_dims(prob1, dim=2), 1)
glimplse2 = tf.reduce_sum(
image_features_flat * tf.expand_dims(prob2, dim=2), 1)
attention_map1 = tf.reshape(prob1, [-1, img_dim, img_dim])
attention_map2 = tf.reshape(prob2, [-1, img_dim, img_dim])
context = tf.concat([glimplse1, glimplse2, encoded_question],
axis=1,
name="context")
print context
print prob1
print prob2
return context, attention_map1, attention_map2
def decoder(self, input_): options = self.options curr_input = input_ for layer_no, dilation in enumerate(options['decoder_dilations']): layer_output = self.decode_layer(curr_input, dilation, layer_no) curr_input = layer_output processed_output = ops.conv1d(tf.nn.relu(layer_output), options['n_target_quant'], name = 'decoder_post_processing') return processed_output
def encode_question(self, question, model_type="bytenet", train=True):
options = self.options
question_mask = tf.sign(question)
length = tf.cast(tf.reduce_sum(question_mask, 1), tf.int32)
question_embedding = tf.nn.embedding_lookup(self.w_question_embedding,
question,
name="question_embedding")
question_embedding = tf.nn.tanh(question_embedding)
dropout_keep_prob = 1
if train:
dropout_keep_prob = options['dropout_keep_prob']
# question_embedding = tf.nn.dropout(question_embedding, dropout_keep_prob)
if model_type == "bytenet":
curr_input = tf.nn.relu(
ops.conv1d(question_embedding,
2 * options['residual_channels'],
name="question_scale_conv"))
for layer_no, dilation in enumerate(options['dilations']):
curr_input = ops.byetenet_residual_block(
curr_input,
dilation,
layer_no,
options['residual_channels'],
options['filter_width'],
causal=True,
train=train)
model_output = curr_input
model_output = ops.last_seq_element(model_output, length)
elif model_type == "lstm":
cell = tf.nn.rnn_cell.LSTMCell(
num_units=options['residual_channels'], state_is_tuple=True)
cell = tf.nn.rnn_cell.DropoutWrapper(
cell, output_keep_prob=dropout_keep_prob)
# cell = tf.nn.rnn_cell.MultiRNNCell([cell] * 2, state_is_tuple=True)
outputs, last_states = tf.nn.dynamic_rnn(cell=cell,
dtype=tf.float32,
sequence_length=length,
inputs=question_embedding)
model_output = ops.last_seq_element(outputs, length)
return model_output
def decoder(self, input_, encoder_embedding=None):
options = self.options
curr_input = input_
if encoder_embedding != None:
# CONDITION WITH ENCODER EMBEDDING FOR THE TRANSLATION MODEL
curr_input = tf.concat(2, [input_, encoder_embedding])
print "Decoder Input", curr_input
for layer_no, dilation in enumerate(options['decoder_dilations']):
layer_output = self.decode_layer(curr_input, dilation, layer_no)
curr_input = layer_output
processed_output = ops.conv1d(tf.nn.relu(layer_output),
options['n_target_quant'],
name='decoder_post_processing')
return processed_output
def decoder(self, input_, encoder_embedding=None):
options = self.options
curr_input = input_
for layer_no, dilation in enumerate(options['decoder_dilations']):
layer_output = self.decode_layer(curr_input, dilation, layer_no)
# FOR TRANSLATION MODEL - add the input embedding after the first layer
if encoder_embedding != None and layer_no == 0:
print "Conditioning Encoder"
layer_output = layer_output + encoder_embedding
curr_input = layer_output
processed_output = ops.conv1d(tf.nn.relu(layer_output),
options['n_target_quant'],
name='decoder_post_processing')
return processed_output
def predict_graph(self, reuse=False):
if reuse:
tf.get_variable_scope().reuse_variables()
context_seq_en = self.itemseq_input_en
context_seq_de = self.itemseq_input_de
dilate_input = self.model_graph(context_seq_en, context_seq_de, train=False)
model_para = self.model_para
if self.is_negsample:
logits_2D = tf.reshape(dilate_input[:, -1:, :], [-1, self.embedding_width])
logits_2D = tf.matmul(logits_2D, self.softmax_w, transpose_b=True)
logits_2D = tf.nn.bias_add(logits_2D, self.softmax_b)
else:
logits = ops.conv1d(tf.nn.relu(dilate_input)[:, -1:, :], model_para['item_size'], name='logits')
logits_2D = tf.reshape(logits, [-1, model_para['item_size']])
probs_flat = tf.nn.softmax(logits_2D)
# self.g_probs = tf.reshape(probs_flat, [-1, tf.shape(self.input_predict)[1], model_para['item_size']])
self.log_probs = probs_flat
def encoder(self, input_):
options = self.options
curr_input = input_
for layer_no, dilation in enumerate(self.options['encoder_dilations']):
layer_output = self.encode_layer(curr_input, dilation, layer_no)
# ENCODE ONLY TILL THE INPUT LENGTH, conditioning should be 0 beyond that
layer_output = tf.mul(layer_output, self.source_masked, name = 'layer_{}_output'.format(layer_no))
curr_input = layer_output
# TO BE CONCATENATED WITH TARGET EMBEDDING
processed_output = tf.nn.relu( ops.conv1d(tf.nn.relu(layer_output),
options['residual_channels'],
name = 'encoder_post_processing') )
processed_output = tf.mul(processed_output, self.source_masked_d, name = 'encoder_processed')
return processed_output
def train_graph(self, is_negsample=False):
self.itemseq_input = tf.placeholder('int32', [None, None],
name='itemseq_input')
label_seq, dilate_input = self.model_graph(self.itemseq_input,
train=True)
model_para = self.model_para
if is_negsample:
logits_2D = tf.reshape(dilate_input,
[-1, model_para['dilated_channels']])
self.softmax_w = tf.get_variable(
"softmax_w",
[model_para['item_size'], model_para['dilated_channels']],
tf.float32, tf.random_normal_initializer(0.0, 0.01))
self.softmax_b = tf.get_variable("softmax_b",
[model_para['item_size']],
tf.float32,
tf.constant_initializer(0.1))
label_flat = tf.reshape(
label_seq, [-1, 1]) # 1 is the number of positive example
num_sampled = int(
0.2 * model_para['item_size']) #sample 20% as negatives
# tf.nn.nce_loss
loss = tf.nn.sampled_softmax_loss(self.softmax_w, self.softmax_b,
label_flat, logits_2D,
num_sampled,
model_para['item_size'])
else:
logits = ops.conv1d(tf.nn.relu(dilate_input),
model_para['item_size'],
name='logits')
logits_2D = tf.reshape(logits, [-1, model_para['item_size']])
label_flat = tf.reshape(label_seq, [-1])
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=label_flat, logits=logits_2D)
self.loss = tf.reduce_mean(loss)
#regularization = 0.001 * tf.reduce_mean([tf.nn.l2_loss(v) for v in tf.trainable_variables()])
#self.loss = self.loss + regularization
self.arg_max_prediction = tf.argmax(
logits_2D, 1
) #useless, if using negative sampling (i.e., negsample=True), it should be changed such as in predict_graph module
def predict_graph_onrecall(self, is_negsample=False, reuse=False):
if reuse:
tf.get_variable_scope().reuse_variables()
self.input_predict = tf.placeholder('int32', [None, None],
name='input_predict')
self.input_recall = tf.placeholder(
'int32', [None, None], name='input_recall') # candidate items
label_seq, dilate_input = self.model_graph(self.input_predict,
train=False)
model_para = self.model_para
if is_negsample:
logits_2D = dilate_input[:, -1:, :]
recall_mat = tf.nn.embedding_lookup(self.softmax_w,
self.input_recall)
logits_2D = tf.matmul(logits_2D,
tf.transpose(recall_mat, [0, 2, 1]))
logits_2D = tf.reshape(logits_2D,
[-1, tf.shape(self.input_recall)[1]])
recall_bias = tf.nn.embedding_lookup(self.softmax_b,
self.input_recall)
logits_2D = tf.add(logits_2D, recall_bias)
else:
# logits = ops.conv1d(tf.nn.relu(dilate_input), model_para['item_size'], name='logits')
logits = ops.conv1d(tf.nn.relu(dilate_input[:, -1:, :]),
model_para['item_size'],
name='logits')
logits_2D = tf.reshape(logits, [-1, model_para['item_size']])
probs_flat = tf.nn.softmax(logits_2D, name='softmax')
self.g_probs = probs_flat
# newly added for weishi, since each input is one user (i.e., a batch), in fact we just need to rank the first batch, the below code is to select top-5
# self.top_k= tf.nn.top_k(self.g_probs[:,-1], k=model_para['top_k'],name='top-k')
#be carefule with the top-k values since the index represents the orders of your recalled items but not the original order.
self.top_k = tf.nn.top_k(self.g_probs,
k=model_para['top_k'],
name='top-k')
def build_model(self):
options = self.options
self.source_sentence = tf.placeholder('int32',
[None, None], name = 'source_sentence')
self.target_sentence = tf.placeholder('int32',
[None, None], name = 'target_sentence')
target_1 = self.target_sentence[:,0:-1]
target_2 = self.target_sentence[:,1:]
source_embedding = tf.nn.embedding_lookup(self.w_source_embedding,
self.source_sentence, name = "source_embedding")
target_1_embedding = tf.nn.embedding_lookup(self.w_target_embedding,
target_1, name = "target_1_embedding")
curr_input = source_embedding
for layer_no, dilation in enumerate(options['encoder_dilations']):
curr_input = ops.byetenet_residual_block(curr_input, dilation,
layer_no, options['residual_channels'],
options['encoder_filter_width'], causal = False, train = True)
encoder_output = curr_input
combined_embedding = target_1_embedding + encoder_output
curr_input = combined_embedding
for layer_no, dilation in enumerate(options['decoder_dilations']):
curr_input = ops.byetenet_residual_block(curr_input, dilation,
layer_no, options['residual_channels'],
options['decoder_filter_width'], causal = True, train = True)
logits = ops.conv1d(tf.nn.relu(curr_input),
options['target_vocab_size'], name = 'logits')
print "logits", logits
logits_flat = tf.reshape(logits, [-1, options['target_vocab_size']])
target_flat = tf.reshape(target_2, [-1])
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels = target_flat, logits = logits_flat)
self.loss = tf.reduce_mean(loss)
self.arg_max_prediction = tf.argmax(logits_flat, 1)
tf.summary.scalar('loss', self.loss)
def discriminator(wave_in):
hi = wave_in
# hi = tf.expand_dims(wave_in, -1)
# batch_size = int(wave_in.get_shape()[0])
# set up the disc_block function
# with tf.variable_scope('d_model') as scope:
# if reuse:
# scope.reuse_variables()
def disc_block(block_idx, input_, kwidth, nfmaps, bnorm, activation, name, pooling=2):
# with tf.variable_scope('d_block_{}'.format(block_idx)):
bias_init = None
if bias_D_conv:
bias_init = tf.constant_initializer(0.)
downconv_init = tf.truncated_normal_initializer(stddev=0.02)
##########################################
hi_a = downconv(input_, nfmaps, kwidth=kwidth, pool=pooling,
init=downconv_init, bias_init=bias_init, name=name)
##########################################
# if bnorm:
# if not reuse:
# print('Applying VBN', end=' *** ')
# hi_a = vbn(hi_a, 'd_vbn_{}'.format(block_idx))
if activation == 'leakyrelu':
hi = leakyrelu(hi_a)
elif activation == 'relu':
hi = tf.nn.relu(hi_a)
else:
raise ValueError('Unrecognized activation {} '
'in D'.format(activation))
return hi
#%%
# beg_size = canvas_size
# apply input noisy layer to real and fake samples
hi = gaussian_noise_layer(hi, disc_noise_std)
for block_idx, fmaps in enumerate(d_num_fmaps):
hi = disc_block(block_idx, hi, 31, d_num_fmaps[block_idx], False, 'leakyrelu',
name='db_{}_{}'.format(block_idx,fmaps))
# hi_f = flatten(hi) #keeps batch size, flatten everything else
#hi_f = tf.nn.dropout(hi_f, self.keep_prob_var)
d_logit_out = conv1d(hi, kwidth=1, num_kernels=1,
init=tf.truncated_normal_initializer(stddev=0.02),
name='logits_conv')
d_logit_out = tf.squeeze(d_logit_out) #removes dimensions of 1
d_logit_out = flatten(d_logit_out)
# all logits connected to 1 single neuron for binary classification
try:
with tf.variable_scope('fc', reuse=True) as fc_sc:
tf.get_variable_scope(). reuse_variables()
d_logit_out = fully_connected(d_logit_out, 1, activation_fn=None, scope=fc_sc)
print('FC_reused')
except ValueError:
with tf.variable_scope('fc') as fc_sc:
d_logit_out = fully_connected(d_logit_out, 1, activation_fn=None, scope=fc_sc)
print('FC created')
return d_logit_out
#%%
#disable_vbn= False
#
#def vbn(tensor, name):
# if disable_vbn:
# class Dummy(object):
# # Do nothing here, no bnorm
# def __init__(self, tensor, ignored):
# self.reference_output=tensor
# def __call__(self, x):
# return x
# VBN_cls = Dummy
# else:
# VBN_cls = VBN
# if not hasattr(self, name):
# vbn = VBN_cls(tensor, name)
# setattr(self, name, vbn)
# return vbn.reference_output
# vbn = getattr(self, name)
# return vbn(tensor)
#%% Testing blocks
#
#a=tf.random_normal([100, 2**14, 1])
#
#b, skippies=encoder(a, True, 'enc')
#
#c= decoder(b, skippies, 'dec')
#
#d=discriminator(a)
def discriminator(wave_in, reuse=True):
hi = wave_in
hi = tf.expand_dims(wave_in, -1)
# batch_size = int(wave_in.get_shape()[0])
# set up the disc_block function
with tf.variable_scope('d_model') as scope:
if reuse:
scope.reuse_variables()
def disc_block(block_idx,
input_,
kwidth,
nfmaps,
bnorm,
activation,
pooling=2):
with tf.variable_scope('d_block_{}'.format(block_idx)):
if not reuse:
print('D block {} input shape: {}'
''.format(block_idx, input_.get_shape()),
end=' *** ')
bias_init = None
if bias_D_conv:
if not reuse:
print('biasing D conv', end=' *** ')
bias_init = tf.constant_initializer(0.)
downconv_init = tf.truncated_normal_initializer(stddev=0.02)
##########################################
hi_a = downconv(input_,
nfmaps,
kwidth=kwidth,
pool=pooling,
init=downconv_init,
bias_init=bias_init)
##########################################
if not reuse:
print('downconved shape: {} '
''.format(hi_a.get_shape()),
end=' *** ')
if bnorm:
if not reuse:
print('Applying VBN', end=' *** ')
hi_a = vbn(hi_a, 'd_vbn_{}'.format(block_idx))
if activation == 'leakyrelu':
if not reuse:
print('Applying Lrelu', end=' *** ')
hi = leakyrelu(hi_a)
elif activation == 'relu':
if not reuse:
print('Applying Relu', end=' *** ')
hi = tf.nn.relu(hi_a)
else:
raise ValueError('Unrecognized activation {} '
'in D'.format(activation))
return hi
#%%
# beg_size = canvas_size
# apply input noisy layer to real and fake samples
hi = gaussian_noise_layer(hi, disc_noise_std)
if not reuse:
print('*** Discriminator summary ***')
for block_idx, fmaps in enumerate(d_num_fmaps):
hi = disc_block(block_idx, hi, 31, d_num_fmaps[block_idx], True,
'leakyrelu')
if not reuse:
print()
if not reuse:
print('discriminator deconved shape: ', hi.get_shape())
hi_f = flatten(hi) #keeps batch size, flatten everything else
#hi_f = tf.nn.dropout(hi_f, self.keep_prob_var)
d_logit_out = conv1d(hi,
kwidth=1,
num_kernels=1,
init=tf.truncated_normal_initializer(stddev=0.02),
name='logits_conv')
d_logit_out = tf.squeeze(d_logit_out) #removes dimensions of 1
# all logits connected to 1 single neuron for binary classification
d_logit_out = fully_connected(d_logit_out, 1, activation_fn=None)
if not reuse:
print('discriminator output shape: ', d_logit_out.get_shape())
print('*****************************')
return d_logit_out
def res_block(inp, dilation, k_w, causal, name): h = ops.conv1d(inp, 128, dilation=dilation, k_w=k_w, causal=causal, name=name) h = tf.nn.relu(h) return h + inp
def forward(self,
x,
meta_loss=None,
meta_step_size=None,
stop_gradient=False):
x = conv1d(inputs=x,
weight=self.conv1.weight,
bias=self.conv1.bias,
meta_step_size=meta_step_size,
meta_loss=meta_loss,
stop_gradient=stop_gradient)
x = self.bn1(x)
x = self.act(x)
x = self.pooling(x)
x = self.dropout(x)
x = conv1d(inputs=x,
weight=self.conv2.weight,
bias=self.conv2.bias,
meta_step_size=meta_step_size,
meta_loss=meta_loss,
stop_gradient=stop_gradient)
x = self.bn2(x)
x = self.act(x)
x = self.pooling(x)
x = self.dropout(x)
x = conv1d(inputs=x,
weight=self.conv3.weight,
bias=self.conv3.bias,
meta_step_size=meta_step_size,
meta_loss=meta_loss,
stop_gradient=stop_gradient)
x = self.bn3(x)
x = self.act(x)
x = self.pooling(x)
x = self.dropout(x)
x = torch.mean(x, dim=2)
x = linear(inputs=x,
weight=self.fc1.weight,
bias=self.fc1.bias,
meta_loss=meta_loss,
meta_step_size=meta_step_size,
stop_gradient=stop_gradient)
x = self.act(x)
x = self.dropout(x)
x = linear(inputs=x,
weight=self.fc2.weight,
bias=self.fc2.bias,
meta_loss=meta_loss,
meta_step_size=meta_step_size,
stop_gradient=stop_gradient)
x = self.act(x)
x = self.dropout(x)
x = linear(inputs=x,
weight=self.fc3.weight,
bias=self.fc3.bias,
meta_loss=meta_loss,
meta_step_size=meta_step_size,
stop_gradient=stop_gradient)
end_points = {'Predictions': F.softmax(input=x, dim=-1)}
return x, end_points
def build_translator(self, reuse=False):
if reuse:
tf.get_variable_scope().reuse_variables()
options = self.options
self.t_source_sentence = tf.placeholder('int32', [None, None],
name='source_sentence')
self.t_target_sentence = tf.placeholder('int32', [None, None],
name='target_sentence')
source_embedding = tf.nn.embedding_lookup(self.w_source_embedding,
self.t_source_sentence,
name="source_embedding")
target_embedding = tf.nn.embedding_lookup(self.w_target_embedding,
self.t_target_sentence,
name="target_embedding")
curr_input = source_embedding
for layer_no, dilation in enumerate(options['encoder_dilations']):
if options['layer_norm']:
curr_input = ops.byetenet_residual_block(
curr_input,
dilation,
layer_no,
options['residual_channels'],
options['encoder_filter_width'],
causal=False,
train=False)
else:
curr_input = ops.byetenet_residual_block_without_ln(
curr_input,
dilation,
layer_no,
options['residual_channels'],
options['encoder_filter_width'],
causal=False,
train=False)
encoder_output = curr_input[:,
0:tf.shape(self.t_target_sentence)[1], :]
combined_embedding = target_embedding + encoder_output
curr_input = combined_embedding
for layer_no, dilation in enumerate(options['decoder_dilations']):
curr_input = ops.byetenet_residual_block(
curr_input,
dilation,
layer_no,
options['residual_channels'],
options['decoder_filter_width'],
causal=True,
train=False)
logits = ops.conv1d(tf.nn.relu(curr_input),
options['target_vocab_size'],
name='logits')
logits_flat = tf.reshape(logits, [-1, options['target_vocab_size']])
probs_flat = tf.nn.softmax(logits_flat)
self.t_probs = tf.reshape(
probs_flat,
[-1, tf.shape(logits)[1], options['target_vocab_size']])
def discriminator(self, input_, cond_):
if self.norm_type == "batch_norm":
from ops import batch_norm as norm
elif self.norm_type == "instance_norm":
from ops import instance_norm as norm
else:
raise Exception("Unknown normalization layer!!!")
if not self.norm_type == "instance_norm":
input_ = tf.nn.dropout(input_, self.kp_)
if self.d_arch == 0:
h0 = lrelu(conv1d(input_, 128, k_w=4, name="conv1d_h0"))
h1 = lrelu(norm(conv1d(h0, 256, k_w=4, name="conv1d_h1"), "bn1"))
h1 = tf.concat([h1, tf.tile(cond_, [1, int(h1.shape[1]), 1])], axis=-1)
h2 = lrelu(norm(conv1d(h1, 512, k_w=4, name="conv1d_h2"), "bn2"))
h2 = tf.concat([h2, tf.tile(cond_, [1, int(h2.shape[1]), 1])], axis=-1)
h3 = lrelu(norm(conv1d(h2, 1024, k_w=4, name="conv1d_h3"), "bn3"))
h3 = tf.concat([h3, tf.tile(cond_, [1, int(h3.shape[1]), 1])], axis=-1)
logits = conv1d(h3, 1, k_w=4, name="logits", padding="VALID")
elif self.d_arch == 1:
h0 = lrelu(conv1d(input_, 64, k_w=4, name="conv1d_h0"))
h1 = lrelu(norm(conv1d(h0, 128, k_w=4, name="conv1d_h1"), "bn1"))
h1 = tf.concat([h1, tf.tile(cond_, [1, int(h1.shape[1]), 1])], axis=-1)
h2 = lrelu(norm(conv1d(h1, 256, k_w=4, name="conv1d_h2"), "bn2"))
h2 = tf.concat([h2, tf.tile(cond_, [1, int(h2.shape[1]), 1])], axis=-1)
h3 = lrelu(norm(conv1d(h2, 512, k_w=4, name="conv1d_h3"), "bn3"))
h3 = tf.concat([h3, tf.tile(cond_, [1, int(h3.shape[1]), 1])], axis=-1)
logits = conv1d(h3, 1, k_w=4, name="logits", padding="VALID")
elif self.d_arch == 2:
h0 = lrelu(conv1d(input_, 32, k_w=4, name="conv1d_h0"))
h1 = lrelu(norm(conv1d(h0, 64, k_w=4, name="conv1d_h1"), "bn1"))
h1 = tf.concat([h1, tf.tile(cond_, [1, int(h1.shape[1]), 1])], axis=-1)
h2 = lrelu(norm(conv1d(h1, 128, k_w=4, name="conv1d_h2"), "bn2"))
h2 = tf.concat([h2, tf.tile(cond_, [1, int(h2.shape[1]), 1])], axis=-1)
h3 = lrelu(norm(conv1d(h2, 256, k_w=4, name="conv1d_h3"), "bn3"))
h3 = tf.concat([h3, tf.tile(cond_, [1, int(h3.shape[1]), 1])], axis=-1)
logits = conv1d(h3, 1, k_w=4, name="logits", padding="VALID")
elif self.d_arch == 3:
h0 = lrelu(conv1d(input_, 16, k_w=4, name="conv1d_h0"))
h1 = lrelu(norm(conv1d(h0, 32, k_w=4, name="conv1d_h1"), "bn1"))
h1 = tf.concat([h1, tf.tile(cond_, [1, int(h1.shape[1]), 1])], axis=-1)
h2 = lrelu(norm(conv1d(h1, 64, k_w=4, name="conv1d_h2"), "bn2"))
h2 = tf.concat([h2, tf.tile(cond_, [1, int(h2.shape[1]), 1])], axis=-1)
h3 = lrelu(norm(conv1d(h2, 128, k_w=4, name="conv1d_h3"), "bn3"))
h3 = tf.concat([h3, tf.tile(cond_, [1, int(h3.shape[1]), 1])], axis=-1)
logits = conv1d(h3, 1, k_w=4, name="logits", padding="VALID")
else:
raise Exception("Unknown discriminator architecture!!!")
return tf.reshape(logits, [self.batch_size, 1])