def sg_summary_audio(tensor, sample_rate=16000, prefix=None):
# defaults
prefix = '' if prefix is None else prefix + '/'
# summary name
name = prefix + _pretty_name(tensor)
# summary statistics
with tf.name_scope('summary'):
tf.audio_summary(name, tensor, sample_rate)
def sg_summary_image(tensor, prefix=None):
# defaults
prefix = '' if prefix is None else prefix + '/'
# summary name
name = prefix + _pretty_name(tensor)
# summary statistics
with tf.name_scope('summary'):
tf.image_summary(name, tensor)
def sg_summary_gradient(tensor, gradient, prefix='50. gradient'):
# defaults
prefix = '' if prefix is None else prefix + '/'
# summary name
name = prefix + _pretty_name(tensor)
# summary statistics
with tf.name_scope('summary'):
tf.scalar_summary(name + '/norm', tf.global_norm([gradient]))
tf.histogram_summary(name, gradient)
def sg_summary_param(tensor, prefix='40. parameters'):
# defaults
prefix = '' if prefix is None else prefix + '/'
# summary name
name = prefix + _pretty_name(tensor)
# summary statistics
with tf.name_scope('summary'):
tf.scalar_summary(name + '/norm', tf.global_norm([tensor]))
tf.histogram_summary(name, tensor)
def sg_summary_metric(tensor, prefix='20. metric'):
# defaults
prefix = '' if prefix is None else prefix + '/'
# summary name
name = prefix + _pretty_name(tensor)
# summary statistics
with tf.name_scope('summary'):
tf.scalar_summary(name + '/avg', tf.reduce_mean(tensor))
tf.histogram_summary(name, tensor)
def load_dir(fp):
"""Load raw paths data into arrays and returns important info
Args:
fp : string path to data
Returns:
Returns array of loaded files
loaded[0] = sound
loaded[1] = text
loaded[2] = dataset size
"""
with tf.name_scope('raw_data'):
ind = 0
raw_audio = []
text = []
for __file in glob.iglob(fp + '/*.*'):
if dataset == 'TIMIT':
if not ("SA" in __file):
ind += 1
if (".wav" in __file):
raw_audio.append(__file)
__targ = __file[:-4] + str('.TXT')
with open(__targ) as f:
for line in f:
res = ''.join(
[i for i in line if not i.isdigit()])
res = (list(res[2:-1].lower().translate(
None, string.punctuation)))
tmp_res = []
for r in res:
if r == ' ':
tmp_res.append(0)
else:
tmp_res.append(ord(r) - 96)
text.append(tmp_res)
else:
ind += 1
if (".trans.txt" in __file):
with open(__file) as f:
_prefix = __file[:-10]
for line in f:
_id = line.split(' ')[0][-5:]
_line = line.split(' ', 1)[-1]
raw_audio.append(_prefix + _id + '.wav')
res = (list(_line[:-1].lower().translate(
None, string.punctuation)))
tmp_res = []
for r in res:
if r == ' ':
tmp_res.append(0)
else:
tmp_res.append(ord(r) - 96)
#print(_prefix+_id+'.wav',decode_to_chars(tmp_res))
text.append(tmp_res)
print(time.strftime('[%H:%M:%S]'),
'Succesfully loaded data set of size', len(raw_audio))
return raw_audio, text, len(raw_audio)
def sg_summary_metric(tensor, prefix='20. metric'):
r"""Writes the average of `tensor` (=metric such as accuracy).
"""
# defaults
prefix = '' if prefix is None else prefix + '/'
# summary name
name = prefix + _pretty_name(tensor)
# summary statistics
with tf.name_scope('summary'):
tf.scalar_summary(name + '/avg', tf.reduce_mean(tensor))
tf.histogram_summary(name, tensor)
def sg_summary_activation(tensor, prefix='30. activation'):
# defaults
prefix = '' if prefix is None else prefix + '/'
# summary name
name = prefix + _pretty_name(tensor)
# summary statistics
with tf.name_scope('summary'):
tf.scalar_summary(name + '/norm', tf.global_norm([tensor]))
tf.scalar_summary(
name + '/ratio',
tf.reduce_mean(tf.cast(tf.greater(tensor, 0), tf.sg_floatx)))
tf.histogram_summary(name, tensor)
def sg_summary_gradient(tensor, gradient, prefix='50. gradient'):
r"""Writes the normalized gradient value
Args:
tensor: A `Tensor` variable.
gradient: A `Tensor`. Gradient of `tensor`.
"""
# defaults
prefix = '' if prefix is None else prefix + '/'
# summary name
name = prefix + _pretty_name(tensor)
# summary statistics
with tf.name_scope('summary'):
try:
tf.scalar_summary(name + '/norm', tf.global_norm([gradient]))
tf.histogram_summary(name, gradient)
except:
pass
def wrapper(**kwargs):
r"""Manages arguments of `tf.sg_opt`.
Args:
kwargs: keyword arguments. The wrapped function will be provided with gpu_index argument.
"""
# parse option
opt = tf.sg_opt(kwargs)
# loop for all available GPUs
res = []
for i in range(sg_gpus()):
# specify device
with tf.device('/gpu:%d' % i):
# give new scope only to operation
with tf.name_scope('gpu_%d' % i):
# save reuse flag
with sg_context(reuse=(True if i > 0 else False)):
# call function
res.append(func(opt * tf.sg_opt(gpu_index=i)))
return res
def sg_optim(loss, **kwargs):
r"""Applies gradients to variables.
Args:
loss: A 0-D `Tensor` containing the value to minimize. list of 0-D tensor for Multiple GPU
kwargs:
optim: A name for optimizer. 'MaxProp' (default), 'AdaMax', 'Adam', 'RMSProp' or 'sgd'.
lr: A Python Scalar (optional). Learning rate. Default is .001.
beta1: A Python Scalar (optional). Default is .9.
beta2: A Python Scalar (optional). Default is .99.
momentum : A Python Scalar for RMSProp optimizer (optional). Default is 0.
category: A string or string list. Specifies the variables that should be trained (optional).
Only if the name of a trainable variable starts with `category`, it's value is updated.
Default is '', which means all trainable variables are updated.
"""
opt = tf.sg_opt(kwargs)
# default training options
opt += tf.sg_opt(optim='MaxProp', lr=0.001, beta1=0.9, beta2=0.99, momentum=0., category='')
# select optimizer
if opt.optim == 'MaxProp':
optim = tf.sg_optimize.MaxPropOptimizer(learning_rate=opt.lr, beta2=opt.beta2)
elif opt.optim == 'AdaMax':
optim = tf.sg_optimize.AdaMaxOptimizer(learning_rate=opt.lr, beta1=opt.beta1, beta2=opt.beta2)
elif opt.optim == 'Adam':
optim = tf.train.AdamOptimizer(learning_rate=opt.lr, beta1=opt.beta1, beta2=opt.beta2)
elif opt.optim == 'RMSProp':
optim = tf.train.RMSPropOptimizer(learning_rate=opt.lr, decay=opt.beta1, momentum=opt.momentum)
else:
optim = tf.train.GradientDescentOptimizer(learning_rate=opt.lr)
# get trainable variables
if isinstance(opt.category, (tuple, list)):
var_list = []
for cat in opt.category:
var_list.extend([t for t in tf.trainable_variables() if t.name.startswith(cat)])
else:
var_list = [t for t in tf.trainable_variables() if t.name.startswith(opt.category)]
#
# calc gradient
#
# multiple GPUs case
if isinstance(loss, (tuple, list)):
gradients = []
# loop for each GPU tower
for i, loss_ in enumerate(loss):
# specify device
with tf.device('/gpu:%d' % i):
# give new scope only to operation
with tf.name_scope('gpu_%d' % i):
# add gradient calculation operation for each GPU tower
gradients.append(tf.gradients(loss_, var_list))
# averaging gradient
gradient = []
for grad in zip(*gradients):
gradient.append(tf.add_n(grad) / len(loss))
# single GPU case
else:
gradient = tf.gradients(loss, var_list)
gradient, _ = tf.clip_by_global_norm(gradient, opt.clip_grad_norm)
# gradient update op
with tf.device('/gpu:0'):
grad_var = [(g, v) for g, v in zip(gradient, var_list)]
grad_op = optim.apply_gradients(grad_var, global_step=tf.sg_global_step())
# add summary using last tower value
for g, v in grad_var:
# exclude batch normal statics
if 'mean' not in v.name and 'variance' not in v.name \
and 'beta' not in v.name and 'gamma' not in v.name:
tf.sg_summary_gradient(v, g)
# extra update ops within category ( for example, batch normal running stat update )
if isinstance(opt.category, (tuple, list)):
update_op = []
for cat in opt.category:
update_op.extend([t for t in tf.get_collection(tf.GraphKeys.UPDATE_OPS) if t.name.startswith(cat)])
else:
update_op = [t for t in tf.get_collection(tf.GraphKeys.UPDATE_OPS) if t.name.startswith(opt.category)]
return tf.group(*([grad_op] + update_op))
if input_noise and not test:
x += np.random.normal(scale=noise_magnitude, size=x.shape)
return x, lengths
####################
with tf.device("/device:GPU:0"):
print(time.strftime('[%H:%M:%S]'), 'Loading network functions... ')
graph = tf.Graph()
with graph.as_default():
def lstm_cell():
with tf.name_scope('cell'):
return tf.contrib.rnn.LSTMCell(num_hidden, state_is_tuple=True)
with tf.name_scope('inputLength'):
seq_len = tf.placeholder(tf.int32, [None])
with tf.name_scope('input'):
inputs = tf.placeholder(tf.float32, [None, None, num_mfccs * 2])
targets = tf.sparse_placeholder(tf.int32)
# Stacking rnn cells
with tf.name_scope('cellStack'):
stack = tf.contrib.rnn.MultiRNNCell(
[lstm_cell() for _ in range(num_layers)], state_is_tuple=True)
outputs, _ = tf.nn.dynamic_rnn(stack,
inputs,
seq_len,
dtype=tf.float32)
shape = tf.shape(inputs)
def generate():
dev = '/cpu:0'
with tf.device(dev):
mydir = 'tfrc150char_wrd0704'
files = [f for f in listdir(mydir) if isfile(join(mydir, f))]
tfrecords_filename = []
tfrecords_filename = [join(mydir, 'short_infer3.tfrecords')
] #[join(mydir, f) for f in tfrecords_filename]
tfrecords_filename_inf = [join(mydir, '11_3.tfrecords')]
print(tfrecords_filename)
filename_queue = tf.train.string_input_producer(tfrecords_filename,
num_epochs=num_epochs,
shuffle=True,
capacity=1)
infer_queue = tf.train.string_input_producer(tfrecords_filename_inf,
num_epochs=num_epochs,
shuffle=True,
capacity=1)
optim = tf.train.AdamOptimizer(learning_rate=0.0001,
beta1=0.9,
beta2=0.99)
# Calculate the gradients for each model tower.
tower_grads = []
reuse_vars = False
with tf.variable_scope("dec_lstm") as scp:
dec_cell = BasicLSTMCell2(Hp.w_emb_size,
Hp.rnn_hd,
state_is_tuple=True)
with tf.variable_scope("contx_lstm") as scp:
cell = BasicLSTMCell2(Hp.hd, Hp.rnn_hd, state_is_tuple=True)
rnn_cell = tf.contrib.rnn.DropoutWrapper(
cell,
input_keep_prob=Hp.keep_prob,
output_keep_prob=Hp.keep_prob)
(words, chars) = read_and_decode(filename_queue,
Hp.batch_size * Hp.num_gpus)
words_splits = tf.split(axis=0,
num_or_size_splits=Hp.num_gpus,
value=words)
chars_splits = tf.split(axis=0,
num_or_size_splits=Hp.num_gpus,
value=chars)
word_emb = np.loadtxt("glove300d_0704.txt")
Hp.word_vs = word_emb.shape[0]
# --------------------------------------------------------------------------------
with tf.name_scope('%s_%d' % ("tower", 0)) as scope:
rnn_state = tower_infer_enc(chars_splits[0],
scope,
rnn_cell,
dec_cell,
word_emb,
out_reuse_vars=False,
dev='/cpu:0')
chars_pl = tf.placeholder(tf.int32, shape=(None, Hp.c_maxlen))
rnn_state_pl1 = [
tf.placeholder(tf.float32, shape=(None, Hp.rnn_hd)),
tf.placeholder(tf.float32, shape=(None, Hp.rnn_hd))
]
rnn_state_pl = tf.contrib.rnn.LSTMStateTuple(
rnn_state_pl1[0], rnn_state_pl1[1])
final_ids, rnn_state_dec = tower_infer_dec(chars_pl,
scope,
rnn_cell,
dec_cell,
word_emb,
rnn_state_pl,
out_reuse_vars=False,
dev='/cpu:0')
# --------------------------------------------------------------------------------
saver = tf.train.Saver(tf.trainable_variables())
session_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
session_config.gpu_options.per_process_gpu_memory_fraction = 0.94
session_config.gpu_options.allow_growth = False
restore_dir = 'tnsrbrd/hin17d08m_1313g2' # lec30d07m_1634g2 lec04d07m_2006g2 lec28d07m_1221g2 lec31d07m_1548g2
csv_file = join(restore_dir, time.strftime("hin%dd%mm_%H%M.csv"))
csv_f = open(csv_file, 'a')
csv_writer = csv.writer(csv_f)
with tf.Session(config=session_config) as sess:
sess.run(
tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer()))
tf.train.start_queue_runners(sess=sess)
saver.restore(sess,
tf.train.latest_checkpoint(
join(restore_dir,
'last_chpt'))) # lec04d07m_2006g2
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for ep in range(num_epochs):
tf.sg_set_infer(sess)
rnn_state_val, w_txt, ch_txt = sess.run(
[rnn_state, words_splits[0], chars_splits[0]],
feed_dict={Hp.keep_prob: 1.0})
predictions = [] #[w_txt[:,2,:]]
for idx in range(3):
char_inpt = word2char_ids(
ids_val) if idx != 0 else ch_txt[:, 2, :]
ids_val, rnn_state_val = sess.run(
[final_ids, rnn_state_dec],
feed_dict={
Hp.keep_prob: 1.0,
rnn_state_pl1[0]: rnn_state_val[0],
rnn_state_pl1[1]: rnn_state_val[1],
chars_pl: char_inpt
})
temp = np.zeros((Hp.batch_size, Hp.w_maxlen))
for b in range(Hp.batch_size):
stop_ind = np.where(ids_val[b] == 2)[0]
if stop_ind.size > 0:
stop_ind = stop_ind[0]
ids_val[b, stop_ind +
1:] = ids_val[b, stop_ind + 1:] * 0
temp[:, :ids_val.shape[1]] = ids_val
predictions.append(temp)
# predictions are decode_sent x b x w_maxlen
predictions = np.array(predictions)
in_batches = [w_txt[b, :, :] for b in range(Hp.batch_size)]
res_batches = [
predictions[:, b, :] for b in range(Hp.batch_size)
]
for b in range(Hp.batch_size):
in_paragraph = idxword2txt(in_batches[b])
print("\n INPUT SAMPLE \n")
print(in_paragraph)
res_paragraph = idxword2txt(res_batches[b])
print("\n RESULTS \n")
print(res_paragraph)
csv_writer.writerow([
" ".join(in_paragraph[:3]), " ".join(in_paragraph[3:]),
" ".join(res_paragraph)
])
csv_f.close()
#loss = logit.sg_ctc(target=y, seq_len=seq_len)
reg_lambda = 0.0002
trainable = tf.trainable_variables()
lossL2 = tf.add_n([tf.nn.l2_loss(v) for v in trainable]) * reg_lambda
loss = logit.sg_ce(target=y, one_hot=True) + lossL2
# train
config = tf.ConfigProto(allow_soft_placement=True,
inter_op_parallelism_threads=6,
intra_op_parallelism_threads=6)
sess = tf.Session(config=config)
tf.sg_init(sess)
learning_rate = tf.train.exponential_decay(0.00001,
tf.sg_global_step(),
100,
0.95,
staircase=False)
with tf.name_scope('summaries'):
tf.summary.scalar('global_step', tf.sg_global_step())
tf.summary.scalar('real_lr', learning_rate)
tf.sg_train(log_interval=30,
lr=learning_rate,
loss=loss,
ep_size=data.num_batch,
max_ep=8,
early_stop=False,
lr_reset=True)
def wrapper(**kwargs):
opt = tf.sg_opt(kwargs)
# default training options
opt += tf.sg_opt(lr=0.001,
save_dir='asset/train',
max_ep=1000,
ep_size=100000,
save_interval=600,
log_interval=60,
early_stop=True,
lr_reset=False,
eval_metric=[],
max_keep=5,
keep_interval=1,
tqdm=True,
console_log=False)
# make directory if not exist
if not os.path.exists(opt.save_dir + '/log'):
os.makedirs(opt.save_dir + '/log')
if not os.path.exists(opt.save_dir + '/ckpt'):
os.makedirs(opt.save_dir + '/ckpt')
# find last checkpoint
last_file = tf.train.latest_checkpoint(opt.save_dir + '/ckpt')
if last_file:
ep = start_ep = int(last_file.split('-')[1]) + 1
start_step = int(last_file.split('-')[2])
else:
ep = start_ep = 1
start_step = 0
# checkpoint saver
saver = tf.train.Saver(max_to_keep=opt.max_keep,
keep_checkpoint_every_n_hours=opt.keep_interval)
# summary writer
summary_writer = tf.train.SummaryWriter(opt.save_dir + '/log',
graph=tf.get_default_graph())
# add learning rate summary
with tf.name_scope('summary'):
tf.scalar_summary('60. learning_rate/learning_rate',
_learning_rate)
# add evaluation metric summary
for m in opt.eval_metric:
tf.sg_summary_metric(m)
# summary op
summary_op = tf.merge_all_summaries()
# create session
if opt.sess:
sess = opt.sess
else:
# session with multiple GPU support
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
# initialize variables
sg_init(sess)
# restore last checkpoint
if last_file:
saver.restore(sess, last_file)
# set learning rate
if start_ep == 1 or opt.lr_reset:
sess.run(_learning_rate.assign(opt.lr))
# logging
tf.sg_info('Training started from epoch[%03d]-step[%d].' %
(start_ep, start_step))
try:
# start data queue runner
with tf.sg_queue_context(sess):
# set session mode to train
tf.sg_set_train(sess)
# loss history for learning rate decay
loss, loss_prev, early_stopped = None, None, False
# time stamp for saving and logging
last_saved = last_logged = time.time()
# epoch loop
for ep in range(start_ep, opt.max_ep + 1):
# show progressbar
if opt.tqdm:
iterator = tqdm(range(opt.ep_size),
desc='train',
ncols=70,
unit='b',
leave=False)
else:
iterator = range(opt.ep_size)
# batch loop
for _ in iterator:
# call train function
batch_loss = func(sess, opt)
# loss history update
if batch_loss is not None:
if loss is None:
loss = np.mean(batch_loss)
else:
loss = loss * 0.9 + np.mean(batch_loss) * 0.1
# saving
if time.time() - last_saved > opt.save_interval:
last_saved = time.time()
saver.save(sess,
opt.save_dir + '/ckpt/model-%03d' % ep,
write_meta_graph=False,
global_step=sess.run(
tf.sg_global_step()))
# logging
if time.time() - last_logged > opt.log_interval:
last_logged = time.time()
# set session mode to infer
tf.sg_set_infer(sess)
# run evaluation op
if len(opt.eval_metric) > 0:
sess.run(opt.eval_metric)
if opt.console_log: # console logging
# log epoch information
tf.sg_info(
'\tEpoch[%03d:lr=%7.5f:gs=%d] - loss = %s'
% (ep, sess.run(_learning_rate),
sess.run(tf.sg_global_step()),
('NA' if loss is None else '%8.6f' %
loss)))
else: # tensorboard logging
# run logging op
summary_writer.add_summary(
sess.run(summary_op),
global_step=sess.run(tf.sg_global_step()))
# learning rate decay
if opt.early_stop and loss_prev:
# if loss stalling
if loss >= 0.95 * loss_prev:
# early stopping
current_lr = sess.run(_learning_rate)
if current_lr < 5e-6:
early_stopped = True
break
else:
# decrease learning rate by half
sess.run(
_learning_rate.assign(current_lr /
2.))
# update loss history
loss_prev = loss
# revert session mode to train
tf.sg_set_train(sess)
# log epoch information
if not opt.console_log:
tf.sg_info(
'\tEpoch[%03d:lr=%7.5f:gs=%d] - loss = %s' %
(ep, sess.run(_learning_rate),
sess.run(tf.sg_global_step()),
('NA' if loss is None else '%8.6f' % loss)))
if early_stopped:
tf.sg_info('\tEarly stopped ( no loss progress ).')
break
finally:
# save last epoch
saver.save(sess,
opt.save_dir + '/ckpt/model-%03d' % ep,
write_meta_graph=False,
global_step=sess.run(tf.sg_global_step()))
# set session mode to infer
tf.sg_set_infer(sess)
# logging
tf.sg_info('Training finished at epoch[%d]-step[%d].' %
(ep, sess.run(tf.sg_global_step())))
# close session
if opt.sess is None:
sess.close()
index = np.arange(db_size)
np.random.shuffle(index)
ret = index[:batchsize]
index = index[:-batchsize].copy()
return ret
def t_get_indices(batchsize):
index = np.arange(batchsize)
np.random.shuffle(index)
return index
## Training Loop
sd = 1 / np.sqrt(num_features)
with tf.name_scope('input'):
X = tf.placeholder(tf.float32, [None, num_features], name="x_inp")
Y = tf.placeholder(tf.float32, [None, num_classes], name="y_inp")
W_1 = tf.Variable(
tf.random_normal([num_features, n_hidden_units_one], mean=0, stddev=sd))
b_1 = tf.Variable(tf.random_normal([n_hidden_units_one], mean=0, stddev=sd))
h_1 = tf.nn.tanh(tf.matmul(X, W_1) + b_1)
W_2 = tf.Variable(
tf.random_normal([n_hidden_units_one, n_hidden_units_two],
mean=0,
stddev=sd))
b_2 = tf.Variable(tf.random_normal([n_hidden_units_two], mean=0, stddev=sd))
h_2 = tf.nn.tanh(tf.matmul(h_1, W_2) + b_2)
def lstm_cell():
with tf.name_scope('cell'):
return tf.contrib.rnn.LSTMCell(num_hidden, state_is_tuple=True)
