def __call__(self, inputs, state):
"""Most basic RNN:
input is [bs, dim]
output is [bs, dim]
"""
logger.debug(self.name)
with tf.variable_scope(self.name) as scope:
dim = inputs.get_shape()[1]
shift = _variable_on_cpu('shift',
dim,
initializer=tf.zeros_initializer())
scale = _variable_on_cpu('scale',
dim,
initializer=tf.ones_initializer())
logger.debug(shift.name)
batch_mean, batch_var = tf.nn.moments(inputs, [0], name='moments')
ema = tf.train.ExponentialMovingAverage(decay=0.5)
if self.is_train:
ema_apply_op = ema.apply([batch_mean, batch_var])
with tf.control_dependencies([ema_apply_op]):
mean = tf.identity(batch_mean)
var = tf.identity(batch_var)
else:
mean, var = lambda: (ema.average(batch_mean),
ema.average(batch_var))
scope.reuse_variables()
bn = tf.nn.batch_normalization(inputs, mean, var, shift, scale,
self.eps)
return bn, bn
def __init__(self,
num_units,
activation=tf.nn.relu6,
skip_input=True,
use_fp16=False,
name=None):
self._num_units = num_units
self.use_fp16 = use_fp16
self.name = name
self.skip_input = skip_input
logger.debug(self.name)
def conv_layer(x,
input_channels,
num_kernels,
kernel_size,
stride_size,
data_format,
padding='VALID',
with_bias=True,
name=None):
"""conv_layer returns a 2d convolution layer.
param:
input_channels, num_kernels and kernel_size are required
size of kernel and stride are list as: [height, width]
data_format: "NHWC" or "NCHW"
padding refer to tf.nn.conv2d padding
"""
assert input_channels is not None and num_kernels is not None
assert isinstance(kernel_size, list) and len(kernel_size) == 2
assert isinstance(stride_size, list) and len(stride_size) == 2
assert isinstance(name, str)
assert data_format in ["NHWC", "NCHW"]
ic = input_channels
# outpout channel
oc = num_kernels
kh, kw = kernel_size
sh, sw = stride_size
if data_format == "NHWC":
stride = [1, sh, sw, 1]
else:
stride = [1, 1, sh, sw]
logger.debug(name)
with tf.name_scope(name):
wgt = weight_variable('param_weight', [kh, kw, ic, oc])
DEBUG(variable_summaries(wgt))
DEBUG(tf.summary.histogram('pre_' + name, x))
conv = tf.nn.conv2d(x,
wgt,
stride,
padding,
data_format=data_format,
name=name)
DEBUG(tf.summary.histogram('post_' + name, conv))
if with_bias:
bias = bias_variable('param_bias', [oc])
DEBUG(variable_summaries(bias))
conv = tf.nn.bias_add(conv, bias, data_format, 'add_bias')
DEBUG(tf.summary.histogram('post_bias', conv))
return conv
def __init__(self,
num_units,
activation=tf.nn.relu6,
is_train=True,
eps=1e-5,
use_fp16=False,
name=None):
self._num_units = num_units
self.use_fp16 = use_fp16
self.eps = eps
self.name = name
self.is_train = is_train
logger.debug(self.name)
def feed_dict():
'''feed inputs for ds2
'''
start_time = time.time()
dat, lbl_ind, lbl_val, lbl_shp, utt = dataset.next_batch(
ARGS.batch_size)
duration = (time.time() - start_time) * 1000
logger.debug('reading data %.3f ms' % duration)
return {
input_data: dat,
input_label_indice: lbl_ind,
input_label_value: lbl_val,
input_label_shape: lbl_shp,
input_utt_lens: utt
}
def main(_):
set_debug_mode(ARGS.debug)
logger.debug(ARGS)
if tf.gfile.Exists(ARGS.log_dir):
tf.gfile.DeleteRecursively(ARGS.log_dir)
tf.gfile.MakeDirs(ARGS.log_dir)
if not tf.gfile.Exists(ARGS.checkpoint_dir):
tf.gfile.MakeDirs(ARGS.checkpoint_dir)
os.environ["KMP_BLOCKTIME"] = "1"
os.environ["KMP_SETTINGS"] = "1"
os.environ["OMP_NUM_THREADS"] = "32"
os.environ["MKL_NUM_THREADS"] = "32"
os.environ["OMP_DYNAMIC"] = "false"
os.environ["KMP_AFFINITY"] = "granularity=fine,verbose,compact,1,0"
train_loop()
def seq_batch_norm(x, dim, seq_lens, eps=1e-5, name=None, is_train=True):
'''sequence batch normalization
input is [seq, bs, dim]
'''
assert len(x.get_shape()) == 3, 'input should be 3-D'
dtype = INFERENCE_DTYPE
logger.debug(dtype)
with tf.variable_scope(name) as scope:
bncell = SeqBNCell(num_units=dim)
#cell = tf.contrib.rnn.CompiledWrapper(bncell)
#cell = tf.contrib.rnn.RNNCell(bncell)
cell = tf.contrib.rnn.MultiRNNCell([bncell])
seqbn_output, _ = tf.nn.dynamic_rnn(
cell,
x,
sequence_length=seq_lens,
dtype=dtype,
time_major=True, # input is [seq, bs, dim]
swap_memory=True)
return seqbn_output
def __call__(self, inputs, state):
"""Most basic RNN:
input shape is [bs, dim]
state shape is [bs, dim]
output shaep is [bs, dim]
"""
logger.debug(self.name)
assert self.skip_input, 'only support skip input mode yet'
with tf.variable_scope(self.name):
#print "rnn cell input size: ", inputs.get_shape().as_list()
#print "rnn cell state size: ", state.get_shape().as_list()
u_ = _variable_on_cpu('U', [self._num_units, self._num_units],
initializer=tf.zeros_initializer(),
use_fp16=self.use_fp16)
resu = tf.matmul(state, u_)
b_ = _variable_on_cpu('B', [self._num_units],
tf.constant_initializer(0),
use_fp16=self.use_fp16)
output = relu_layer(resu + b_, capping=20)
return output, output
def fc_layer(x, dim_out, dim_in=None, with_bias=True, name=None):
'''fc_layer returns a full connected layer
Wx + b
param:
if dim_in is None, will set as dim_out
'''
logger.debug(name)
if dim_in is None:
dim_in = dim_out
with tf.name_scope(name):
DEBUG(tf.summary.histogram('pre_' + name, x))
x = tf.reshape(x, [-1, dim_in])
wgt = weight_variable('weight', [dim_in, dim_out])
DEBUG(variable_summaries(wgt))
fc_ = tf.matmul(x, wgt)
DEBUG(tf.summary.histogram('post_' + name, fc_))
if with_bias:
bias = bias_variable('bias', [dim_out])
DEBUG(variable_summaries(bias))
fc_ = tf.add(fc_, bias, 'add_bias')
DEBUG(tf.summary.histogram('post_bias' + name, fc_))
return fc_
def train_loop():
'''ds2 train loop
'''
def feed_dict():
'''feed inputs for ds2
'''
start_time = time.time()
dat, lbl_ind, lbl_val, lbl_shp, utt = dataset.next_batch(
ARGS.batch_size)
duration = (time.time() - start_time) * 1000
logger.debug('reading data %.3f ms' % duration)
return {
input_data: dat,
input_label_indice: lbl_ind,
input_label_value: lbl_val,
input_label_shape: lbl_shp,
input_utt_lens: utt
}
with tf.name_scope('inputs'):
# defulat input data shape is [bs, seq, 161]
input_data = tf.placeholder(dtype=tf.float32,
shape=[None, None, Dataset.freq_bins])
input_label_indice = tf.placeholder(dtype=tf.int64, shape=[None, None])
input_label_value = tf.placeholder(dtype=tf.int32, shape=[None])
input_label_shape = tf.placeholder(dtype=tf.int64, shape=[2])
input_utt_lens = tf.placeholder(dtype=tf.int32, shape=[None])
input_label = tf.SparseTensor(indices=input_label_indice,
values=input_label_value,
dense_shape=input_label_shape)
#with tf.name_scope('ds2'):
with tf.variable_scope('ds2'):
loss_op = ds2.get_loss(input_data, input_label, input_utt_lens,
ARGS.data_format)
with tf.name_scope('train'):
optimizer = tf.train.AdamOptimizer(ARGS.learning_rate)
train_op = optimizer.minimize(loss_op)
dataset = Dataset(use_dummy=ARGS.use_dummy)
# Create a saver. TODO: how about only inference
saver = tf.train.Saver(tf.global_variables())
sess = tf.Session(config=tf.ConfigProto(
allow_soft_placement=False, #True,
log_device_placement=False,
inter_op_parallelism_threads=8,
intra_op_parallelism_threads=32))
last_iter = init_variables(sess, saver, ARGS.checkpoint_dir)
run_options = None
run_metadata = None
# Merge all the summaries and write them out to ARGS.log_dir
if ARGS.debug:
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(ARGS.log_dir, sess.graph)
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
#assert not np.isnan(loss_value), 'Model diverged with loss = NaN'
sum_time = 0
min_time = float('inf')
for i in range(ARGS.max_iter):
start_time = time.time()
train_loss, _ = sess.run([loss_op, train_op],
feed_dict=feed_dict(),
options=run_options,
run_metadata=run_metadata)
duration = time.time() - start_time
sum_time += duration
if min_time > duration:
min_time = duration
# Save the model checkpoint periodically
if i % ARGS.checkpoint_iter == 0 or (i + 1) == ARGS.max_iter:
checkpoint_path = ARGS.checkpoint_dir + '/ds2_model'
global_step = last_iter + i
saver.save(sess, checkpoint_path, global_step=global_step)
logger.info('save checkpoint to %s-%d' %
(ARGS.checkpoint_dir, global_step))
if i == ARGS.profil_iter:
# save timeline to a json and only save once
filename = ARGS.log_dir + ('/timeline_iter%d' % i) + '.json'
logger.debug('save timeline to: ' + filename)
save_timeline(filename, run_metadata)
# tfprof
logger.debug('save TFprof to: ' + ARGS.log_dir)
save_tfprof(ARGS.log_dir, sess.graph, run_metadata)
if i % ARGS.loss_iter_interval == 0:
format_str = 'iter %d, training loss = %.2f (%.3f sec/iter)'
log_content = (i, train_loss, duration)
if ARGS.debug:
s = time.time()
summary = sess.run(summary_op,
feed_dict=feed_dict(),
options=run_options,
run_metadata=run_metadata)
summary_writer.add_run_metadata(run_metadata, 'iter%03d' % i)
summary_writer.add_summary(summary, i)
d = time.time() - s
format_str += ', summary %.3g sec/iter'
log_content += (d, )
logger.info(format_str % log_content)
if ARGS.debug:
summary_writer.close()
logger.info('avg time: %.3f ms, min_time: %.3f ms' %
(sum_time * 1000 / ARGS.max_iter, min_time * 1000))
sess.close()
