def train(self): # train baseline model
input_ph = tf.placeholder(shape=[batch_size, 28, 28, 1],
dtype=tf.float32)
label_ph = tf.placeholder(shape=[
batch_size,
], dtype=tf.int32)
predict = self.forward(input_ph)
loss_tensor = tf.reduce_mean(predict.sg_ce(target=label_ph))
# use to update network parameters
optim = tf.sg_optim(loss_tensor, optim='Adam', lr=1e-3)
# use saver to save a new model
saver = tf.train.Saver()
sess = tf.Session()
with tf.sg_queue_context(sess):
# inital
tf.sg_init(sess)
# validation
acc = (predict.sg_reuse(
input=Mnist.valid.image).sg_softmax().sg_accuracy(
target=Mnist.valid.label, name='validation'))
tf.sg_train(loss=loss,
eval_metric=[acc],
max_ep=max_ep,
save_dir=save_dir,
ep_size=Mnist.train.num_batch,
log_interval=10)
def sg_input(shape=None, dtype=sg_floatx, name=None):
if shape is None:
return tf.placeholder(dtype, shape=None, name=name)
else:
if not isinstance(shape, (list, tuple)):
shape = [shape]
return tf.placeholder(dtype, shape=[None] + list(shape), name=name)
def __init__(self, mode="train"):
'''
Args:
mode: A string. Either "train" , "val", or "test"
'''
if mode == 'train':
self.X_batch, self.Y_batch, self.num_batch = get_batch_data(
'train')
else:
self.X_batch = tf.placeholder(
tf.int32, [Hyperparams.batch_size, Hyperparams.maxlen])
self.Y_batch = tf.placeholder(
tf.int32, [Hyperparams.batch_size, Hyperparams.maxlen])
self.X_batch_rev = self.X_batch.sg_reverse_seq() # (8, 100)
# make embedding matrix for input characters
embed_mat = tf.convert_to_tensor(load_embed_lookup_table())
# embed table lookup
X_batch_3d = self.X_batch.sg_lookup(
emb=embed_mat).sg_float() # (8, 100, 200)
X_batch_rev_3d = self.X_batch_rev.sg_lookup(
emb=embed_mat).sg_float() # (8, 100, 200)
# 1st biGRU layer
gru_fw1 = X_batch_3d.sg_gru(dim=Hyperparams.hidden_dim,
ln=True) # (8, 100, 200)
gru_bw1 = X_batch_rev_3d.sg_gru(dim=Hyperparams.hidden_dim,
ln=True) # (8, 100, 200)
gru1 = gru_fw1.sg_concat(target=gru_bw1) # (8, 100, 400)
# 2nd biGRU layer
gru_fw2 = gru1.sg_gru(dim=Hyperparams.hidden_dim * 2,
ln=True) # (8, 100, 400)
gru_bw2 = gru1.sg_gru(dim=Hyperparams.hidden_dim * 2,
ln=True) # (8, 100, 400)
gru2 = gru_fw2.sg_concat(target=gru_bw2) # (16, 100, 800)
# fc dense layer
reshaped = gru2.sg_reshape(shape=[-1, gru2.get_shape().as_list()[-1]])
logits = reshaped.sg_dense(dim=3) # 1 for space 2 for non-space
self.logits = logits.sg_reshape(shape=gru2.get_shape().as_list()[:-1] +
[-1])
if mode == 'train':
# cross entropy loss with logits ( for training set )
self.loss = self.logits.sg_ce(target=self.Y_batch, mask=True)
# accuracy evaluation ( for validation set )
self.X_val_batch, self.Y_val_batch, self.num_batch = get_batch_data(
'val')
self.acc = (self.logits.sg_reuse(
input=self.X_val_batch).sg_accuracy(target=self.Y_val_batch,
name='val'))
def __init__(self):
# set log level to debug
tf.sg_verbosity(10)
# batch size
self.batch_size = 1
# vocabulary size
self.voca_size = sttwdata.voca_size
# mfcc feature of audio
self.x = tf.placeholder(dtype=tf.sg_floatx,
shape=(self.batch_size, None, 20))
# encode audio feature
self.logit = get_logit(self.x, voca_size=self.voca_size)
# sequence length except zero-padding
self.seq_len = tf.not_equal(self.x.sg_sum(axis=2),
0.).sg_int().sg_sum(axis=1)
# run network
self.session = tf.Session()
tf.sg_init(self.session)
self.saver = tf.train.Saver()
self.saver.restore(self.session,
tf.train.latest_checkpoint('asset/train'))
def init_model():
global x, y
# set log level to debug
tf.sg_verbosity(10)
#
# hyper parameters
#
batch_size = 1 # batch size
#
# inputs
#
# vocabulary size
voca_size = data.voca_size
# print(voca_size)
# mfcc feature of audio
x = tf.placeholder(dtype=tf.sg_floatx, shape=(batch_size, None, 20))
# sequence length except zero-padding
seq_len = tf.not_equal(x.sg_sum(axis=2), 0.).sg_int().sg_sum(axis=1)
# encode audio feature
logit = get_logit(x, voca_size=voca_size)
# ctc decoding
decoded, _ = tf.nn.ctc_beam_search_decoder(
logit.sg_transpose(perm=[1, 0, 2]), seq_len, merge_repeated=False)
# to dense tensor
y = tf.sparse_to_dense(decoded[0].indices, decoded[0].dense_shape,
decoded[0].values) + 1
def sg_input(shape=None, dtype=sg_floatx, name=None):
r"""Creates a placeholder.
Args:
shape: A tuple/list of integers. If an integers is given, it will turn to a list.
dtype: A data type. Default is float32.
name: A name for the placeholder.
Returns:
A wrapped placeholder `Tensor`.
"""
if shape is None:
return tf.placeholder(dtype, shape=None, name=name)
else:
if not isinstance(shape, (list, tuple)):
shape = [shape]
return tf.placeholder(dtype, shape=[None] + list(shape), name=name)
def test(self):
print 'Testing model {}: addnoise={}, rotate={}, var={}'.format(
save_dir, addnoise, rotate, var)
input_ph = tf.placeholder(shape=[batch_size, 28, 28, 1],
dtype=tf.float32)
label_ph = tf.placeholder(shape=[
batch_size,
], dtype=tf.int32)
predict = self.forward(input_ph)
acc = (predict.sg_softmax().sg_accuracy(target=label_ph, name='test'))
sess = tf.Session()
with tf.sg_queue_context(sess):
tf.sg_init(sess)
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(save_dir))
total_accuracy = 0
for i in range(Mnist.test.num_batch):
[image_array,
label_array] = sess.run([Mnist.test.image, Mnist.test.label])
if addnoise:
image_array[0, :, :, 0] = addnoisy(image_array[0, :, :, 0],
var)
if rotate:
image_array[0, :, :, 0] = rotate_90(image_array[0, :, :,
0])
acc_value = sess.run([acc],
feed_dict={
input_ph: image_array,
label_ph: label_array
})[0]
total_accuracy += np.sum(acc_value)
print 'Evaluation accuracy: {}'.format(
float(total_accuracy) / (Mnist.test.num_batch * batch_size))
# close session
sess.close()
def wrapper(**kwargs):
r"""Manages arguments of `tf.sg_opt`.
Args:
**kwargs:
source: A source queue list to enqueue
dtypes: Data types of each tensor
capacity: Queue capacity. Default is 32.
num_threads: Number of threads. Default is 1.
"""
# default option
opt = tf.sg_opt(kwargs) + tf.sg_opt(
dtypes=[tf.sg_floatx], capacity=32, num_threads=1)
# source queue list check
assert opt.source is not None, 'source is mandatory.'
if type(opt.source) is not list and type(opt.source) is not tuple:
opt.source = [opt.source]
if type(opt.dtypes) is not list and type(opt.dtypes) is not tuple:
opt.dtypes = [opt.dtypes]
assert len(opt.source) == len(
opt.dtypes), 'Source and dtypes should have same length.'
# enqueue function
def enqueue_func(sess, op):
# read data from source queue
data = func(sess.run(opt.source))
# create feeder dict
feed_dict = {}
for ph, col in zip(placeholders, data):
feed_dict[ph] = col
# run session
sess.run(op, feed_dict=feed_dict)
# create place holder list
placeholders = []
for dtype in opt.dtypes:
placeholders.append(tf.placeholder(dtype=dtype))
# create FIFO queue
queue = tf.FIFOQueue(opt.capacity, dtypes=opt.dtypes)
# enqueue operation
enqueue_op = queue.enqueue(placeholders)
# create queue runner
runner = _FuncQueueRunner(enqueue_func, queue,
[enqueue_op] * opt.num_threads)
# register to global collection
tf.train.add_queue_runner(runner)
# return de-queue operation
return queue.dequeue()
def testIt():
data = raw
positive = np.array(data.label_train) > 0
x = tf.placeholder(tf.float32, [None, 4096])
y = tf.placeholder(tf.float32)
disc_real = discriminator(x)
accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.cast(disc_real > 0.5, "float"), y), tf.float32))
np.set_printoptions(precision=3, suppress=True)
with tf.Session() as sess:
sess.run(
tf.group(tf.global_variables_initializer(),
tf.sg_phase().assign(False)))
# restore parameters
tf.sg_restore(sess,
tf.train.latest_checkpoint('asset/train/gan'),
category=['generator', 'discriminator'])
ans = sess.run(disc_real, feed_dict={x: np.array(data.test)})
print np.sum(ans > 0.5)
np.save('dm_bird.npy', ans)
def __init__(self, mode="train"):
'''
Args:
mode: A string. Either "train" or "test"
'''
self.char2idx, self.idx2char = load_char_vocab()
self.word2idx, self.idx2word = load_word_vocab()
if mode == "train":
self.x, self.y, self.num_batch = get_batch_data()
else:
self.x = tf.placeholder(tf.int32, [None, Hyperparams.seqlen])
self.emb_x = tf.sg_emb(name='emb_x',
voca_size=len(self.char2idx),
dim=Hyperparams.embed_dim)
self.enc = self.x.sg_lookup(emb=self.emb_x)
with tf.sg_context(size=5, act='relu', bn=True):
for _ in range(20):
dim = self.enc.get_shape().as_list()[-1]
self.enc += self.enc.sg_conv1d(
dim=dim) # (64, 50, 300) float32
self.enc = self.enc.sg_conv1d(size=1,
dim=len(self.word2idx),
act='linear',
bn=False) # (64, 50, 21293) float32
# self.logits = self.enc.sg_mean(dims=[1], keep_dims=False) # (64, 21293) float32
# Weighted Sum. Updated on Feb. 15, 2017.
def make_weights(size):
weights = tf.range(1, size + 1, dtype=tf.float32)
weights *= 1. / ((1 + size) * size // 2)
weights = tf.expand_dims(weights, 0)
weights = tf.expand_dims(weights, -1)
return weights
self.weights = make_weights(Hyperparams.seqlen) # (1, 50, 1)
self.enc *= self.weights # Broadcasting
self.logits = self.enc.sg_sum(axis=[1], keep_dims=False) # (64, 21293)
if mode == "train":
self.ce = self.logits.sg_ce(target=self.y,
mask=False,
one_hot=False)
self.istarget = tf.not_equal(self.y, tf.ones_like(
self.y)).sg_float() # 1: Unkown
self.reduced_loss = ((self.ce * self.istarget).sg_sum()) / (
self.istarget.sg_sum() + 1e-5)
tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
class Hp:
batch_size = 51 # batch size
hd = 400 # hidden dimension
c_maxlen = 150 # Maximum sentence length
w_maxlen = 25
char_vocab = u'''␀␂␃⁇N abcdefghijklmnopqrstuvwxyz'''
char_vs = len(char_vocab)
par_maxlen = [3]
num_blocks = 3 # dilated blocks
keep_prob = tf.placeholder(tf.float32)
w_emb_size = 300
num_gpus = num_gpus
rnn_hd = 300
def __init__(self, is_train=True):
# inputs
if is_train:
self.X, self.Y, self.num_batch = get_batch_data(
) # (16, 9, 9, 1), (16, 9, 9)
self.X_val, self.Y_val, _ = get_batch_data(is_train=False)
else:
self.X = tf.placeholder(tf.float32, [None, 9, 9, 1])
with tf.sg_context(size=3, act='relu', bn=True):
self.logits = self.X.sg_identity()
for _ in range(5):
self.logits = (self.logits.sg_conv(dim=512))
self.logits = self.logits.sg_conv(
dim=10, size=1, act='linear',
bn=False) # (16, 9, 9, 10) float32
if is_train:
self.ce = self.logits.sg_ce(target=self.Y,
mask=False) # (16, 9, 9) dtype=float32
self.istarget = tf.equal(
self.X.sg_squeeze(), tf.zeros_like(self.X.sg_squeeze())
).sg_float() # zeros: 1, non-zeros: 0 (16, 9, 9) dtype=float32
self.loss = self.ce * self.istarget # (16, 9, 9) dtype=float32
self.reduced_loss = self.loss.sg_sum() / self.istarget.sg_sum()
tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
# accuracy evaluation ( for train set )
self.preds = (self.logits.sg_argmax()).sg_int()
self.hits = tf.equal(self.preds, self.Y).sg_float()
self.acc_train = (self.hits *
self.istarget).sg_sum() / self.istarget.sg_sum()
# accuracy evaluation ( for validation set )
self.preds_ = (self.logits.sg_reuse(
input=self.X_val).sg_argmax()).sg_int()
self.hits_ = tf.equal(self.preds_, self.Y_val).sg_float()
self.istarget_ = tf.equal(self.X_val.sg_squeeze(),
tf.zeros_like(
self.X_val.sg_squeeze())).sg_float()
self.acc_val = (self.hits_ *
self.istarget_).sg_sum() / self.istarget_.sg_sum()
def __init__(self, mode="train"):
'''
Args:
is_train: Boolean. If True, backprop is executed.
'''
if mode == "train":
self.x, self.y, self.num_batch = get_batch_data(
) # (64, 50) int64, (64, 50) int64, 1636
else: # test
self.x = tf.placeholder(tf.int64, [None, Hyperparams.maxlen])
# make embedding matrix for input characters
pnyn2idx, _, hanzi2idx, _ = load_vocab()
self.emb_x = tf.sg_emb(name='emb_x',
voca_size=len(pnyn2idx),
dim=Hyperparams.embed_dim)
self.enc = self.x.sg_lookup(emb=self.emb_x)
with tf.sg_context(size=5, act='relu', bn=True):
for _ in range(20):
dim = self.enc.get_shape().as_list()[-1]
self.enc += self.enc.sg_conv1d(
dim=dim) # (64, 50, 300) float32
# final fully convolutional layer for softmax
self.logits = self.enc.sg_conv1d(size=1,
dim=len(hanzi2idx),
act='linear',
bn=False) # (64, 50, 5072) float32
if mode == "train":
self.ce = self.logits.sg_ce(target=self.y,
mask=True) # (64, 50) float32
self.istarget = tf.not_equal(self.y, tf.zeros_like(
self.y)).sg_float() # (64, 50) float32
self.reduced_loss = self.ce.sg_sum() / self.istarget.sg_sum(
) # () float32
tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
def __init__(self, is_train=True):
'''
Args:
is_train: Boolean. If True, backprop is executed.
'''
if is_train:
self.x, self.y = get_batch_data() # (16, 100), (16, 100)
else:
# self.x = tf.placeholder(tf.int32, [Hyperparams.batch_size, Hyperparams.maxlen])
self.x = tf.placeholder(tf.int32, [None, Hyperparams.maxlen])
# make embedding matrix for input characters
hangul2idx, _, hanja2idx, _ = load_charmaps()
self.emb_x = tf.sg_emb(name='emb_x', voca_size=len(hangul2idx), dim=Hyperparams.hidden_dim)
# embed table lookup
self.enc = self.x.sg_lookup(emb=self.emb_x).sg_float() # (16, 100, 200)
# loop dilated conv block
for i in range(2):
self.enc = (self.enc
.sg_res_block(size=5, rate=1)
.sg_res_block(size=5, rate=2)
.sg_res_block(size=5, rate=4)
.sg_res_block(size=5, rate=8)
.sg_res_block(size=5, rate=16))
# final fully convolutional layer for softmax
self.logits = self.enc.sg_conv1d(size=1, dim=len(hanja2idx)) # (16, 100, 4543)
if is_train:
self.ce = self.logits.sg_ce(target=self.y, mask=True) # (16, 100)
self.nonzeros = tf.not_equal(self.y, tf.zeros_like(self.y)).sg_float() # (16, 100)
self.reduced_loss = self.ce.sg_sum() / self.nonzeros.sg_sum() # ()
tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
def __init__(self, is_train=True):
# inputs
if is_train:
self.x, self.y, self.num_batch = get_batch_data()
self.x_val, self.y_val, _ = get_batch_data(is_train=False)
else:
self.x = tf.placeholder(tf.float32, [None, 9, 9, 1])
with tf.sg_context(size=3, act='relu', bn=True):
self.logits = self.x.sg_identity()
for _ in range(10):
self.logits = (self.logits.sg_conv(dim=512))
self.logits = self.logits.sg_conv(dim=10,
size=1,
act='linear',
bn=False)
if is_train:
self.ce = self.logits.sg_ce(target=self.y, mask=False)
self.istarget = tf.equal(self.x.sg_squeeze(),
tf.zeros_like(
self.x.sg_squeeze())).sg_float()
self.loss = self.ce * self.istarget
self.reduced_loss = self.loss.sg_sum() / self.istarget.sg_sum()
tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
# accuracy evaluation ( for validation set )
self.preds_ = (self.logits.sg_reuse(
input=self.x_val).sg_argmax()).sg_int()
self.hits_ = tf.equal(self.preds_, self.y_val).sg_float()
self.istarget_ = tf.equal(self.x_val.sg_squeeze(),
tf.zeros_like(
self.x_val.sg_squeeze())).sg_float()
self.acc = (self.hits_ *
self.istarget_).sg_sum() / self.istarget_.sg_sum()
def __init__(self, mode="train"):
# Inputs and Labels
if mode == "train":
self.x, self.y, self.num_batch = get_batch_data(
) # (16, 150) int32, (16, 150) int32, int
self.y_src = tf.concat(
[tf.zeros((Hp.batch_size, 1), tf.int32), self.y[:, :-1]],
1) # (16, 150) int32
else: # inference
self.x = tf.placeholder(tf.int32, shape=(Hp.batch_size, Hp.maxlen))
self.y_src = tf.placeholder(tf.int32,
shape=(Hp.batch_size, Hp.maxlen))
# Load vocabulary
char2idx, idx2char = load_vocab()
# Embedding
emb_x = tf.sg_emb(name='emb_x',
voca_size=len(char2idx),
dim=Hp.hidden_units) # (179, 320)
emb_y = tf.sg_emb(name='emb_y',
voca_size=len(char2idx),
dim=Hp.hidden_units) # (179, 320)
X = self.x.sg_lookup(emb=emb_x) # (16, 150, 320)
Y = self.y_src.sg_lookup(emb=emb_y) # (16, 150, 320)
# Encoding
conv = X.sg_quasi_conv1d(is_enc=True, size=6) # (16*3, 150, 320)
pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320)
H_zfo1 = pool[Hp.batch_size:] # (16*3, 15, 320) for decoding
conv = pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*3, 150, 320)
pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320)
H_zfo2 = pool[Hp.batch_size:] # (16*3, 150, 320) for decoding
conv = pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*3, 150, 320)
pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320)
H_zfo3 = pool[Hp.batch_size:] # (16*3, 150, 320) for decoding
conv = pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*3, 150, 320)
pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320)
H4 = pool[:Hp.batch_size] # (16, 150, 320) for decoding
H_zfo4 = pool[Hp.batch_size:] # (16*3, 150, 320) for decoding
# Decoding
d_conv = (Y.sg_concat(target=H_zfo1,
axis=0).sg_quasi_conv1d(is_enc=False, size=2))
d_pool = d_conv.sg_quasi_rnn(is_enc=False,
att=False) # (16*4, 150, 320)
d_conv = (d_pool.sg_concat(target=H_zfo2,
axis=0).sg_quasi_conv1d(is_enc=False,
size=2))
d_pool = d_conv.sg_quasi_rnn(is_enc=False,
att=False) # (16*4, 150, 320)
d_conv = (d_pool.sg_concat(target=H_zfo3,
axis=0).sg_quasi_conv1d(is_enc=False,
size=2))
d_pool = d_conv.sg_quasi_rnn(is_enc=False,
att=False) # (16*4, 150, 320)
d_conv = (d_pool.sg_concat(target=H_zfo4,
axis=0).sg_quasi_conv1d(is_enc=False,
size=2))
concat = H4.sg_concat(target=d_conv, axis=0)
d_pool = concat.sg_quasi_rnn(is_enc=False, att=True) # (16, 150, 320)
logits = d_pool.sg_conv1d(size=1, dim=len(char2idx),
act="linear") # (16, 150, 179)
if mode == 'train':
# cross entropy loss with logits ( for training set )
loss = logits.sg_ce(target=self.y, mask=True)
istarget = tf.not_equal(self.y, 0).sg_float()
self.reduced_loss = (loss.sg_sum()) / (istarget.sg_sum() + 0.00001)
tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
else: # inference
self.preds = logits.sg_argmax()
def generate():
dev = '/cpu:0'
with tf.device(dev):
mydir = 'tfrc150char_wrd0704'
files = [f for f in listdir(mydir) if isfile(join(mydir, f))]
tfrecords_filename = []
tfrecords_filename = [join(mydir, 'short_infer3.tfrecords')
] #[join(mydir, f) for f in tfrecords_filename]
tfrecords_filename_inf = [join(mydir, '11_3.tfrecords')]
print(tfrecords_filename)
filename_queue = tf.train.string_input_producer(tfrecords_filename,
num_epochs=num_epochs,
shuffle=True,
capacity=1)
infer_queue = tf.train.string_input_producer(tfrecords_filename_inf,
num_epochs=num_epochs,
shuffle=True,
capacity=1)
optim = tf.train.AdamOptimizer(learning_rate=0.0001,
beta1=0.9,
beta2=0.99)
# Calculate the gradients for each model tower.
tower_grads = []
reuse_vars = False
with tf.variable_scope("dec_lstm") as scp:
dec_cell = BasicLSTMCell2(Hp.w_emb_size,
Hp.rnn_hd,
state_is_tuple=True)
with tf.variable_scope("contx_lstm") as scp:
cell = BasicLSTMCell2(Hp.hd, Hp.rnn_hd, state_is_tuple=True)
rnn_cell = tf.contrib.rnn.DropoutWrapper(
cell,
input_keep_prob=Hp.keep_prob,
output_keep_prob=Hp.keep_prob)
(words, chars) = read_and_decode(filename_queue,
Hp.batch_size * Hp.num_gpus)
words_splits = tf.split(axis=0,
num_or_size_splits=Hp.num_gpus,
value=words)
chars_splits = tf.split(axis=0,
num_or_size_splits=Hp.num_gpus,
value=chars)
word_emb = np.loadtxt("glove300d_0704.txt")
Hp.word_vs = word_emb.shape[0]
# --------------------------------------------------------------------------------
with tf.name_scope('%s_%d' % ("tower", 0)) as scope:
rnn_state = tower_infer_enc(chars_splits[0],
scope,
rnn_cell,
dec_cell,
word_emb,
out_reuse_vars=False,
dev='/cpu:0')
chars_pl = tf.placeholder(tf.int32, shape=(None, Hp.c_maxlen))
rnn_state_pl1 = [
tf.placeholder(tf.float32, shape=(None, Hp.rnn_hd)),
tf.placeholder(tf.float32, shape=(None, Hp.rnn_hd))
]
rnn_state_pl = tf.contrib.rnn.LSTMStateTuple(
rnn_state_pl1[0], rnn_state_pl1[1])
final_ids, rnn_state_dec = tower_infer_dec(chars_pl,
scope,
rnn_cell,
dec_cell,
word_emb,
rnn_state_pl,
out_reuse_vars=False,
dev='/cpu:0')
# --------------------------------------------------------------------------------
saver = tf.train.Saver(tf.trainable_variables())
session_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
session_config.gpu_options.per_process_gpu_memory_fraction = 0.94
session_config.gpu_options.allow_growth = False
restore_dir = 'tnsrbrd/hin17d08m_1313g2' # lec30d07m_1634g2 lec04d07m_2006g2 lec28d07m_1221g2 lec31d07m_1548g2
csv_file = join(restore_dir, time.strftime("hin%dd%mm_%H%M.csv"))
csv_f = open(csv_file, 'a')
csv_writer = csv.writer(csv_f)
with tf.Session(config=session_config) as sess:
sess.run(
tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer()))
tf.train.start_queue_runners(sess=sess)
saver.restore(sess,
tf.train.latest_checkpoint(
join(restore_dir,
'last_chpt'))) # lec04d07m_2006g2
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for ep in range(num_epochs):
tf.sg_set_infer(sess)
rnn_state_val, w_txt, ch_txt = sess.run(
[rnn_state, words_splits[0], chars_splits[0]],
feed_dict={Hp.keep_prob: 1.0})
predictions = [] #[w_txt[:,2,:]]
for idx in range(3):
char_inpt = word2char_ids(
ids_val) if idx != 0 else ch_txt[:, 2, :]
ids_val, rnn_state_val = sess.run(
[final_ids, rnn_state_dec],
feed_dict={
Hp.keep_prob: 1.0,
rnn_state_pl1[0]: rnn_state_val[0],
rnn_state_pl1[1]: rnn_state_val[1],
chars_pl: char_inpt
})
temp = np.zeros((Hp.batch_size, Hp.w_maxlen))
for b in range(Hp.batch_size):
stop_ind = np.where(ids_val[b] == 2)[0]
if stop_ind.size > 0:
stop_ind = stop_ind[0]
ids_val[b, stop_ind +
1:] = ids_val[b, stop_ind + 1:] * 0
temp[:, :ids_val.shape[1]] = ids_val
predictions.append(temp)
# predictions are decode_sent x b x w_maxlen
predictions = np.array(predictions)
in_batches = [w_txt[b, :, :] for b in range(Hp.batch_size)]
res_batches = [
predictions[:, b, :] for b in range(Hp.batch_size)
]
for b in range(Hp.batch_size):
in_paragraph = idxword2txt(in_batches[b])
print("\n INPUT SAMPLE \n")
print(in_paragraph)
res_paragraph = idxword2txt(res_batches[b])
print("\n RESULTS \n")
print(res_paragraph)
csv_writer.writerow([
" ".join(in_paragraph[:3]), " ".join(in_paragraph[3:]),
" ".join(res_paragraph)
])
csv_f.close()
config.gpu_options.allow_growth = True #allocate dynamically
# PARAMS #
num_layers = 3
activation = tf.nn.relu
batchsize = 8
w_dropout = 0.4
path_to_train_data = "__data/stage1_train"
path_to_test_data = "__data/stage1_test"
img_dat = util.crawl_path(path_to_train_data)
graph = tf.Graph()
with graph.as_default():
#batchsize*x_max*y_max*rgb(3)
features = tf.placeholder(tf.float32, [None, None, None, 3])
labels = tf.placeholder(tf.int32, [None, None])
input_layer = tf.reshape(features, [-1, 2048, 2048, 3])
training = tf.placeholder(tf.bool)
conv1 = tf.layers.conv2d(inputs=input_layer,
filters=32,
kernel_size=[11, 11],
padding="same",
activation=tf.nn.relu)
print('conv1', conv1.get_shape())
pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[4, 4], strides=2)
print('pool1', pool1.get_shape())
conv2 = tf.layers.conv2d(inputs=input_layer,
filters=64,
kernel_size=[5, 5],
res = (tensor.sg_dense(dim=1024, name='fc1').sg_dense(
dim=7 * 7 * 128,
name='fc2').sg_reshape(shape=(-1, 7, 7, 128)).sg_upconv(
dim=64, name='conv1').sg_upconv(dim=1,
act='sigmoid',
bn=False,
name='conv2'))
return res
#
# inputs
#
# target_number
target_num = tf.placeholder(dtype=tf.sg_intx, shape=batch_size)
# target continuous variable # 1
target_cval_1 = tf.placeholder(dtype=tf.sg_floatx, shape=batch_size)
# target continuous variable # 2
target_cval_2 = tf.placeholder(dtype=tf.sg_floatx, shape=batch_size)
# category variables
z = (tf.ones(batch_size, dtype=tf.sg_intx) *
target_num).sg_one_hot(depth=cat_dim)
# continuous variables
z = z.sg_concat(
target=[target_cval_1.sg_expand_dims(),
target_cval_2.sg_expand_dims()])
# random seed = categorical variable + continuous variable + random normal
from model import *
import data
import sys
# mengatur log level untuk debug
tf.sg_verbosity(10)
# hyper parameters
batch_size = 1 # batch size
# inputs
# panjang kata
voca_size = data.voca_size
# menginput mfcc feature pada file audio
x = tf.placeholder(dtype=tf.sg_floatx, shape=(batch_size, None, 20))
# panjang sequence kecuali zero-padding
seq_len = tf.not_equal(x.sg_sum(axis=2), 0.).sg_int().sg_sum(axis=1)
# encode audio feature
logit = get_logit(x, voca_size=voca_size)
# ctc decoding
decoded, _ = tf.nn.ctc_beam_search_decoder(logit.sg_transpose(perm=[1, 0, 2]),
seq_len,
merge_repeated=False)
# to dense tensor
y = tf.sparse_to_dense(decoded[0].indices, decoded[0].dense_shape,
decoded[0].values) + 1
Author: JJ Fu
"""
import sugartensor as tf
import numpy as np
import librosa
import tensorflow as tfw
from tensorflow.python.framework import graph_util
from model import *
import data
batch_size = 1 # batch size
voca_size = data.voca_size
x = tf.placeholder(dtype=tf.sg_floatx, shape=(batch_size, None, 20))
# sequence length except zero-padding
seq_len = tf.not_equal(x.sg_sum(axis=2), 0.).sg_int().sg_sum(axis=1)
# encode audio feature
logit = get_logit(x, voca_size)
# ctc decoding
decoded, _ = tf.nn.ctc_beam_search_decoder(logit.sg_transpose(perm=[1, 0, 2]), seq_len, merge_repeated=False)
# to dense tensor
y = tf.add(tf.sparse_to_dense(decoded[0].indices, decoded[0].dense_shape, decoded[0].values), 1, name="output")
with tf.Session() as sess:
tf.sg_init(sess)
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint('asset/train'))
graph = tf.get_default_graph()
# # hyper parameters # batch_size = 10 # # inputs # # ComTrans parallel corpus input tensor ( with QueueRunner ) data = ComTrans(batch_size=batch_size) # place holders x = tf.placeholder(dtype=tf.sg_intx, shape=(batch_size, data.max_len)) y_in = tf.placeholder(dtype=tf.sg_intx, shape=(batch_size, data.max_len)) # vocabulary size voca_size = data.voca_size # make embedding matrix for source and target emb_x = tf.sg_emb(name='emb_x', voca_size=voca_size, dim=latent_dim) emb_y = tf.sg_emb(name='emb_y', voca_size=voca_size, dim=latent_dim) # latent from embed table z_x = x.sg_lookup(emb=emb_x) z_y = y_in.sg_lookup(emb=emb_y) # encode graph ( atrous convolution ) enc = encode(z_x)
def __init__(self,
x,
y,
num_batch,
vocab_size,
emb_dim,
hidden_dim,
max_ep=240,
infer_shape=(1, 1),
mode="train"):
self.num_batch = num_batch
self.emb_dim = emb_dim
self.hidden_dim = hidden_dim
self.vocab_size = vocab_size
self.max_len_infer = 512
self.max_ep = max_ep
# reuse = len([t for t in tf.global_variables() if t.name.startswith('gen')]) > 0
reuse = (mode == 'infer')
if mode == "train":
self.x = x
self.y = y
elif mode == "infer":
self.x = tf.placeholder(tf.int32, shape=infer_shape)
self.y = tf.placeholder(tf.int32, shape=infer_shape)
with tf.variable_scope("gen_embs", reuse=reuse):
self.emb_x = tf.get_variable("emb_x",
[self.vocab_size, self.emb_dim])
self.emb_y = tf.get_variable("emb_y",
[self.vocab_size, self.emb_dim])
self.X = tf.nn.embedding_lookup(self.emb_x, self.x)
self.Y = tf.nn.embedding_lookup(self.emb_y, self.y)
with tf.sg_context(name='gen', reuse=reuse):
# self.emb_x = tf.Variable(tf.random_uniform([self.vocab_size, self.emb_dim], 0.0, 1.0), name="emb_x")
# self.emb_y = tf.Variable(tf.random_uniform([self.vocab_size, self.emb_dim], 0.0, 1.0), name="emb_y")
# self.emb_x = tf.sg_emb(name='emb_x', voca_size=self.vocab_size, dim=self.emb_dim) # (68,16)
# self.emb_y = tf.sg_emb(name='emb_y', voca_size=self.vocab_size, dim=self.emb_dim) # (68,16)
# self.X = self.x.sg_lookup(emb=self.emb_x) # (8,63,16)
# self.Y = self.y.sg_lookup(emb=self.emb_y) # (8,63,16)
if mode == "train":
self.lstm_layer = self.X.sg_lstm(in_dim=self.emb_dim,
dim=self.vocab_size,
name="lstm") # (8, 63, 68)
self.test = self.lstm_layer.sg_softmax(name="testtt")
print "mazum??"
print self.test
elif mode == "infer":
self.lstm_layer = self.X.sg_lstm(in_dim=self.emb_dim,
dim=self.vocab_size,
last_only=True,
name="lstm")
self.log_prob = tf.log(self.lstm_layer)
# next_token: select by distribution probability, preds: select by argmax
self.multinormed = tf.multinomial(self.log_prob, 1)
self.next_token = tf.cast(
tf.reshape(tf.multinomial(self.log_prob, 1),
[1, infer_shape[0]]), tf.int32)
self.preds = self.lstm_layer.sg_argmax()
if mode == "train":
self.loss = self.lstm_layer.sg_ce(target=self.y)
self.istarget = tf.not_equal(self.y, 0).sg_float()
self.reduced_loss = (self.loss.sg_sum()) / (
self.istarget.sg_sum() + 0.0000001)
tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
tf.sg_verbosity(10)
#
# hyper parameters
#
batch_size = 100 # batch size
#
# inputs
#
# target continuous variable
target_cval = []
for _ in range(con_dim):
target_cval.append(tf.placeholder(dtype=tf.sg_floatx, shape=batch_size))
# continuous variables
z = target_cval[0].sg_expand_dims()
for i in range(1, con_dim):
z = z.sg_concat(target=target_cval[i].sg_expand_dims())
# random seed = continuous variable + random normal
z = z.sg_concat(target=tf.random_normal((batch_size, rand_dim)))
# generator
gen = generator(z).sg_squeeze(axis=(2, 3))
#
# run generator
def __init__(self, mode="train"):
# Inputs and Labels
if mode == "train":
self.x, self.y, self.num_batch = get_batch_data(
) # (16, 150) int32, (16, 150) int32, int
self.y_src = tf.concat(
axis=1,
values=[tf.zeros((Hp.bs, 1), tf.int32),
self.y[:, :-1]]) # (16, 150) int32
else: # inference
self.x = tf.placeholder(tf.int32, shape=(Hp.bs, Hp.maxlen))
self.y_src = tf.placeholder(tf.int32, shape=(Hp.bs, Hp.maxlen))
# Load vocabulary
self.char2idx, self.idx2char = load_vocab()
# Embedding
self.emb_x = tf.sg_emb(name='emb_x',
voca_size=len(self.char2idx),
dim=Hp.hd) # (179, 320)
self.emb_y = tf.sg_emb(name='emb_y',
voca_size=len(self.char2idx),
dim=Hp.hd) # (179, 320)
self.X = self.x.sg_lookup(emb=self.emb_x) # (16, 150, 320)
self.Y = self.y_src.sg_lookup(emb=self.emb_y) # (16, 150, 320)
# Encoding
self.conv = self.X.sg_quasi_conv1d(is_enc=True,
size=6) # (16*4, 150, 320)
self.pool = self.conv.sg_quasi_rnn(is_enc=True,
att=False) # (16*4, 150, 320)
self.H_zfo1 = self.pool[Hp.bs:] # (16*3, 15, 320) for decoding
self.conv = self.pool.sg_quasi_conv1d(is_enc=True,
size=2) # (16*4, 150, 320)
self.pool = self.conv.sg_quasi_rnn(is_enc=True,
att=False) # (16*4, 150, 320)
self.H_zfo2 = self.pool[Hp.bs:] # (16*3, 150, 320) for decoding
self.conv = self.pool.sg_quasi_conv1d(is_enc=True,
size=2) # (16*4, 150, 320)
self.pool = self.conv.sg_quasi_rnn(is_enc=True,
att=False) # (16*4, 150, 320)
self.H_zfo3 = self.pool[Hp.bs:] # (16*3, 150, 320) for decoding
self.conv = self.pool.sg_quasi_conv1d(is_enc=True,
size=2) # (16*4, 150, 320)
self.pool = self.conv.sg_quasi_rnn(is_enc=True,
att=False) # (16*4, 150, 320)
self.H4 = self.pool[:Hp.bs]
self.H_zfo4 = self.pool[Hp.bs:] # (16*3, 150, 320) for decoding
# Decoding
self.dec = self.Y.sg_concat(target=self.H_zfo1, dim=0)
self.d_conv = self.dec.sg_quasi_conv1d(is_enc=False, size=2)
self.d_pool = self.d_conv.sg_quasi_rnn(is_enc=False,
att=False) # (16*4, 150, 320)
self.d_conv = (self.d_pool.sg_concat(
target=self.H_zfo2, dim=0).sg_quasi_conv1d(is_enc=False, size=2))
self.d_pool = self.d_conv.sg_quasi_rnn(is_enc=False,
att=False) # (16*4, 150, 320)
self.d_conv = (self.d_pool.sg_concat(
target=self.H_zfo3, dim=0).sg_quasi_conv1d(is_enc=False, size=2))
self.d_pool = self.d_conv.sg_quasi_rnn(is_enc=False,
att=False) # (16*4, 150, 320)
self.d_conv = (self.d_pool.sg_concat(
target=self.H_zfo4, dim=0).sg_quasi_conv1d(is_enc=False, size=2))
self.concat = self.H4.sg_concat(target=self.d_conv, dim=0)
self.d_pool = self.concat.sg_quasi_rnn(is_enc=False,
att=True) # (16*4, 150, 320)
self.logits = self.d_pool.sg_conv1d(size=1,
dim=len(self.char2idx),
act="linear") # (16, 150, 179)
self.preds = self.logits.sg_argmax()
if mode == 'train':
# cross entropy loss with logits ( for training set )
self.loss = self.logits.sg_ce(target=self.y, mask=True)
self.istarget = tf.not_equal(self.y, 0).sg_float()
self.reduced_loss = (self.loss.sg_sum()) / (
self.istarget.sg_sum() + 0.00001)
tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
#corpus = SpeechCorpus(batch_size=batch_size * tf.sg_gpus())
mfccs = []
# for mfcc_file in corpus.mfcc_file:
# mfcc = np.load(mfcc_file, allow_pickle=False)
# mfccs.append(mfcc.reshape((1, mfcc.shape[0], mfcc.shape[1])).transpose([0,2,1]))
for mfcc_file in ["more_data/mfcc-one/rainbow.wav.npy"]:
mfcc = np.load(mfcc_file, allow_pickle=False)
mfccs.append(
mfcc.reshape((1, mfcc.shape[0], mfcc.shape[1])).transpose([0, 2, 1]))
# vocabulary size
voca_size = data.voca_size
# mfcc feature of audio
x = tf.placeholder(dtype=tf.sg_floatx, shape=(batch_size, None, 20))
noise = tf.get_variable("noise",
shape=(batch_size, mfccs[index].shape[1], 20),
initializer=tf.zeros_initializer())
perturbed_input = x + noise
# sequence length except zero-padding
seq_len = tf.not_equal(x.sg_sum(axis=2), 0.).sg_int().sg_sum(axis=1)
# encode audio feature
logit = get_logit(perturbed_input, voca_size=voca_size)
# ctc decoding
decoded, _ = tf.nn.ctc_beam_search_decoder(logit.sg_transpose(perm=[1, 0, 2]),
def train_with_GP(self):
input_ph = tf.placeholder(shape=[batch_size, 28, 28, 1],
dtype=tf.float32)
label_ph = tf.placeholder(shape=[
batch_size,
], dtype=tf.int32)
predict = self.forward(input_ph)
loss_tensor = tf.reduce_mean(predict.sg_ce(target=label_ph))
# use to update network parameters
optim = tf.sg_optim(loss_tensor, optim='Adam', lr=1e-3)
# use saver to save a new model
saver = tf.train.Saver()
sess = tf.Session()
with tf.sg_queue_context(sess):
# inital
tf.sg_init(sess)
# train by GP guilding
for e in range(max_ep):
previous_loss = None
for i in range(Mnist.train.num_batch):
[image_array, label_array
] = sess.run([Mnist.train.image, Mnist.train.label])
if (e == 0 or e == 1
): # first and second epoch train no noisy image
loss = sess.run([loss_tensor, optim],
feed_dict={
input_ph: image_array,
label_ph: label_array
})[0]
print 'Baseline loss = ', loss
elif (
e == 2
): # third epoch train with gp image and original image
gpIn1 = np.squeeze(image_array)
gpIn2 = np.zeros((28, 28))
image_gp = GP(gpIn1, gpIn2, seed=0.8)
image_gp2 = image_gp[np.newaxis, ...]
image_gp2 = image_gp2[..., np.newaxis]
loss = sess.run([loss_tensor, optim],
feed_dict={
input_ph: image_array,
label_ph: label_array
})[0]
print 'GP without nosiy loss = ', loss
loss = sess.run([loss_tensor, optim],
feed_dict={
input_ph: image_gp2,
label_ph: label_array
})[0]
print 'GP loss = ', loss
else: # other epoch train with gp evolution
gpIn1 = np.squeeze(image_array)
gpIn2 = np.zeros((28, 28))
image_gp = GP(gpIn1, gpIn2, seed=random.random())
image_gp2 = image_gp[np.newaxis, ...]
image_gp2 = image_gp2[..., np.newaxis]
loss = sess.run([loss_tensor, optim],
feed_dict={
input_ph: image_array,
label_ph: label_array
})[0]
print 'GP without nosiy loss = ', loss
loss = sess.run([loss_tensor, optim],
feed_dict={
input_ph: image_gp2,
label_ph: label_array
})[0]
print 'GP loss = ', loss
if loss < previous_loss:
for i in range(5):
loss = sess.run([loss_tensor, optim],
feed_dict={
input_ph: image_gp2,
label_ph: label_array
})[0]
gpIn1 = image_gp2
image_gp2[0, :, :,
0] = GP(gpIn1[0, :, :, 0],
gpIn2,
seed=random.random())
print 'GP EV loss = ', loss
previous_loss = loss
saver.save(sess,
os.path.join(save_dir, 'gp_model'),
global_step=e)
# close session
sess.close()
def __init__(self):
with tf.sg_context(name='generator'):
self.x = tf.sg_initializer.he_uniform(name="x",
shape=[1, 224, 224,
1]) # noise image
self.y = tf.placeholder(dtype=tf.float32,
shape=[1, 224, 224,
1]) # true target image
with tf.sg_context(name='conv', act='relu'):
self.x_conv1 = (self.x.sg_conv(dim=64).sg_conv().sg_pool()
) # (1, 112, 112, 64)
self.x_conv2 = (self.x_conv1.sg_conv(dim=128).sg_conv().sg_pool()
) # (1, 56, 56, 128)
self.x_conv3 = (self.x_conv2.sg_conv(
dim=256).sg_conv().sg_conv().sg_conv().sg_pool()
) # (1, 28, 28, 256)
self.x_conv4 = (self.x_conv3.sg_conv(
dim=512).sg_conv().sg_conv().sg_conv().sg_pool()
) # (1, 14, 14, 512)
# .sg_conv(dim=512)
# .sg_conv()
# .sg_conv()
# .sg_conv()
# .sg_pool())
self.y_conv1 = self.x_conv1.sg_reuse(input=self.y)
self.y_conv2 = self.x_conv2.sg_reuse(input=self.y)
self.y_conv3 = self.x_conv3.sg_reuse(input=self.y)
self.y_conv4 = self.x_conv4.sg_reuse(input=self.y)
#
def get_gram_mat(tensor):
'''
Arg:
tensor: 4-D tensor. The first dimension must be 1.
Returns:
gram matrix. Read `https://en.wikipedia.org/wiki/Gramian_matrix` for details.
512 by 512.
'''
assert tensor.get_shape(
).ndims == 4, "The tensor must be 4 dimensions."
dim0, dim1, dim2, dim3 = tensor.get_shape().as_list()
tensor = tensor.sg_reshape(shape=[dim0 * dim1 * dim2,
dim3]) #(1*7*7, 512)
# normalization: Why? Because the original value of gram mat. would be too huge.
mean, variance = tf.nn.moments(tensor, [0, 1])
tensor = (tensor - mean) / tf.sqrt(variance + tf.sg_eps)
tensor_t = tensor.sg_transpose(perm=[1, 0]) #(512, 1*7*7)
gram_mat = tf.matmul(tensor_t, tensor) # (512, 512)
return gram_mat
# Loss: Add the loss of each layer
self.mse = tf.squared_difference(get_gram_mat(self.x_conv1), get_gram_mat(self.y_conv1)).sg_mean() +\
tf.squared_difference(get_gram_mat(self.x_conv2), get_gram_mat(self.y_conv2)).sg_mean() +\
tf.squared_difference(get_gram_mat(self.x_conv3), get_gram_mat(self.y_conv3)).sg_mean() +\
tf.squared_difference(get_gram_mat(self.x_conv4), get_gram_mat(self.y_conv4)).sg_mean()
self.train_gen = tf.sg_optim(
self.mse, lr=0.0001,
category='generator') # Note that we train only variable x.
# # hyper parameters # batch_size = 100 # batch size num_category = 10 # category variable number num_cont = 2 # continuous variable number num_dim = 30 # total latent dimension ( category + continuous + noise ) # # inputs # # target_number target_num = tf.placeholder(dtype=tf.sg_intx, shape=batch_size) # target continuous variable # 1 target_cval_1 = tf.placeholder(dtype=tf.sg_floatx, shape=batch_size) # target continuous variable # 2 target_cval_2 = tf.placeholder(dtype=tf.sg_floatx, shape=batch_size) # category variables z = (tf.ones(batch_size, dtype=tf.sg_intx) * target_num).sg_one_hot(depth=num_category) # continuous variables z = z.sg_concat(target=[target_cval_1.sg_expand_dims(), target_cval_2.sg_expand_dims()]) # random seed = categorical variable + continuous variable + random uniform z = z.sg_concat(target=tf.random_uniform((batch_size, num_dim-num_cont-num_category)))
x += np.random.normal(scale=noise_magnitude, size=x.shape)
return x, lengths
####################
with tf.device("/device:GPU:0"):
print(time.strftime('[%H:%M:%S]'), 'Loading network functions... ')
graph = tf.Graph()
with graph.as_default():
def lstm_cell():
with tf.name_scope('cell'):
return tf.contrib.rnn.LSTMCell(num_hidden, state_is_tuple=True)
with tf.name_scope('inputLength'):
seq_len = tf.placeholder(tf.int32, [None])
with tf.name_scope('input'):
inputs = tf.placeholder(tf.float32, [None, None, num_mfccs * 2])
targets = tf.sparse_placeholder(tf.int32)
# Stacking rnn cells
with tf.name_scope('cellStack'):
stack = tf.contrib.rnn.MultiRNNCell(
[lstm_cell() for _ in range(num_layers)], state_is_tuple=True)
outputs, _ = tf.nn.dynamic_rnn(stack,
inputs,
seq_len,
dtype=tf.float32)
shape = tf.shape(inputs)
batch_s, TF_max_timesteps = shape[0], shape[1]
def __init__(self, mode="train"):
# Inputs and Labels
if mode == "train":
self.x, self.y, self.num_batch = get_batch_data() # (16, 150) int32, (16, 150) int32, int
self.y_src = tf.concat([tf.zeros((Hp.batch_size, 1), tf.int32), self.y[:, :-1]], 1) # (16, 150) int32
else: # inference
self.x = tf.placeholder(tf.int32, shape=(Hp.batch_size, Hp.maxlen))
self.y_src = tf.placeholder(tf.int32, shape=(Hp.batch_size, Hp.maxlen))
# Load vocabulary
char2idx, idx2char = load_vocab()
# Embedding
def embed(inputs, vocab_size, embed_size, variable_scope):
'''
inputs = tf.expand_dims(tf.range(5), 0) => (1, 5)
_embed(inputs, 5, 10) => (1, 5, 10)
'''
with tf.variable_scope(variable_scope):
lookup_table = tf.get_variable('lookup_table',
dtype=tf.float32,
shape=[vocab_size, embed_size],
initializer=tf.truncated_normal_initializer())
return tf.nn.embedding_lookup(lookup_table, inputs)
X = embed(self.x, vocab_size=len(char2idx), embed_size=Hp.hidden_units, variable_scope='X') # (179, 320)
Y = embed(self.y_src, vocab_size=len(char2idx), embed_size=Hp.hidden_units, variable_scope='Y') # (179, 320)
# Y = tf.concat((tf.zeros_like(Y[:, :1, :]), Y[:, :-1, :]), 1)
# Encoding
conv = X.sg_quasi_conv1d(is_enc=True, size=6) # (16*3, 150, 320)
pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320)
H_zfo1 = pool[Hp.batch_size:] # (16*3, 15, 320) for decoding
conv = pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*3, 150, 320)
pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320)
H_zfo2 = pool[Hp.batch_size:] # (16*3, 150, 320) for decoding
conv = pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*3, 150, 320)
pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320)
H_zfo3 = pool[Hp.batch_size:] # (16*3, 150, 320) for decoding
conv = pool.sg_quasi_conv1d(is_enc=True, size=2) # (16*3, 150, 320)
pool = conv.sg_quasi_rnn(is_enc=True, att=False) # (16*4, 150, 320)
H4 = pool[:Hp.batch_size] # (16, 150, 320) for decoding
H_zfo4 = pool[Hp.batch_size:] # (16*3, 150, 320) for decoding
# Decoding
d_conv = (Y.sg_concat(target=H_zfo1, axis=0)
.sg_quasi_conv1d(is_enc=False, size=2))
d_pool = d_conv.sg_quasi_rnn(is_enc=False, att=False) # (16*4, 150, 320)
d_conv = (d_pool.sg_concat(target=H_zfo2, axis=0)
.sg_quasi_conv1d(is_enc=False, size=2))
d_pool = d_conv.sg_quasi_rnn(is_enc=False, att=False) # (16*4, 150, 320)
d_conv = (d_pool.sg_concat(target=H_zfo3, axis=0)
.sg_quasi_conv1d(is_enc=False, size=2))
d_pool = d_conv.sg_quasi_rnn(is_enc=False, att=False) # (16*4, 150, 320)
d_conv = (d_pool.sg_concat(target=H_zfo4, axis=0)
.sg_quasi_conv1d(is_enc=False, size=2))
concat = H4.sg_concat(target=d_conv, axis=0)
d_pool = concat.sg_quasi_rnn(is_enc=False, att=True) # (16, 150, 320)
logits = d_pool.sg_conv1d(size=1, dim=len(char2idx), act="linear") # (16, 150, 179)
if mode=='train':
# cross entropy loss with logits ( for training set )
self.loss = logits.sg_ce(target=self.y, mask=True)
istarget = tf.not_equal(self.y, 0).sg_float()
self.reduced_loss = (self.loss.sg_sum()) / (istarget.sg_sum() + 1e-8)
tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
else: # inference
self.preds = logits.sg_argmax()
np.random.shuffle(index)
ret = index[:batchsize]
index = index[:-batchsize].copy()
return ret
def t_get_indices(batchsize):
index = np.arange(batchsize)
np.random.shuffle(index)
return index
## Training Loop
sd = 1 / np.sqrt(num_features)
with tf.name_scope('input'):
X = tf.placeholder(tf.float32, [None, num_features], name="x_inp")
Y = tf.placeholder(tf.float32, [None, num_classes], name="y_inp")
W_1 = tf.Variable(
tf.random_normal([num_features, n_hidden_units_one], mean=0, stddev=sd))
b_1 = tf.Variable(tf.random_normal([n_hidden_units_one], mean=0, stddev=sd))
h_1 = tf.nn.tanh(tf.matmul(X, W_1) + b_1)
W_2 = tf.Variable(
tf.random_normal([n_hidden_units_one, n_hidden_units_two],
mean=0,
stddev=sd))
b_2 = tf.Variable(tf.random_normal([n_hidden_units_two], mean=0, stddev=sd))
h_2 = tf.nn.tanh(tf.matmul(h_1, W_2) + b_2)
W_3 = tf.Variable(
