def test_cut_or_padding(self):
# test for 1d
origin_1_t = tf.placeholder(dtype=tf.int32, shape=[None])
after_1_t = cut_or_padding(origin_1_t, 3)
with self.cached_session(use_gpu=False, force_gpu=False) as sess:
# test for padding
res = sess.run(after_1_t, feed_dict={origin_1_t: [1, 2]})
self.assertAllEqual(res, [1, 2, 0])
# test for cut
res = sess.run(after_1_t, feed_dict={origin_1_t: [1, 2, 3, 4, 5]})
self.assertAllEqual(res, [1, 2, 3])
# test for 2d
origin_2_t = tf.placeholder(dtype=tf.int32, shape=[None, None])
after_2_t = cut_or_padding(origin_2_t, 3)
with self.cached_session(use_gpu=False, force_gpu=False) as sess:
# test for padding
res = sess.run(after_2_t, feed_dict={origin_2_t: [[1, 2], [1, 2]]})
self.assertAllEqual(res, [[1, 2, 0], [1, 2, 0]])
# test for cut
res = sess.run(
after_2_t,
feed_dict={origin_2_t: [[1, 2, 3, 4], [1, 2, 3, 4]]})
self.assertAllEqual(res, [[1, 2, 3], [1, 2, 3]])
def export_inputs(self):
"""Inputs for exported model."""
vocab_dict = load_vocab_dict(self.text_vocab_file_path)
vocab_size = len(vocab_dict)
self.config['data']['vocab_size'] = vocab_size
input_sent_left = tf.placeholder(
shape=(None,), dtype=tf.string, name="input_sent_left")
input_sent_right = tf.placeholder(
shape=(None,), dtype=tf.string, name="input_sent_right")
input_pipeline_func = self.get_input_pipeline(for_export=True)
token_ids_left = input_pipeline_func(input_sent_left)
token_ids_right = input_pipeline_func(input_sent_right)
token_ids_len_left = tf.map_fn(
lambda x: compute_sen_lens(x, padding_token=0), token_ids_left)
token_ids_len_right = tf.map_fn(
lambda x: compute_sen_lens(x, padding_token=0), token_ids_right)
export_data = {
"export_inputs": {
"input_sent_left": input_sent_left,
"input_sent_right": input_sent_right,
},
"model_inputs": {
"input_x_left": token_ids_left,
"input_x_right": token_ids_right,
"input_x_left_len": token_ids_len_left,
"input_x_right_len": token_ids_len_right,
"input_x_len": [token_ids_len_left, token_ids_len_right]
}
}
return export_data
def transfer_bert_model(bert_model_dir, output_bert_model):
graph = tf.Graph()
max_seq_len = 512
num_labels = 2
use_one_hot_embeddings = False
with graph.as_default():
with tf.Session() as sess:
input_ids = tf.placeholder(tf.int32, (None, None), 'input_ids')
input_mask = tf.placeholder(tf.int32, (None, None), 'input_mask')
segment_ids = tf.placeholder(tf.int32, (None, None), 'segment_ids')
bert_config = modeling.BertConfig.from_json_file(os.path.join(bert_model_dir, 'bert_config.json'))
model = modeling.BertModel(
config=bert_config,
is_training=False,
input_ids=input_ids,
input_mask=input_mask,
token_type_ids=segment_ids,
use_one_hot_embeddings=use_one_hot_embeddings)
all_encoder_layers = model.get_all_encoder_layers()
input_x_bert_cls = model.get_pooled_output()
for idx, layer in enumerate(all_encoder_layers):
layer = tf.identity(layer, "encoder_layers_" + str(idx))
print("layer:", layer)
input_x_bert_cls = tf.identity(input_x_bert_cls, "input_x_bert_cls")
print("input_x_bert_cls", input_x_bert_cls)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver.restore(sess, bert_model_dir + "/bert_model.ckpt")
saver.save(sess, output_bert_model)
def _freq_feat_graph(feat_name, **kwargs):
winlen = kwargs.get('winlen')
winstep = kwargs.get('winstep')
feature_size = kwargs.get('feature_size')
sr = kwargs.get('sr') #pylint: disable=invalid-name
nfft = kwargs.get('nfft')
del nfft
assert feat_name in ('fbank', 'spec')
params = speech_ops.speech_params(
sr=sr,
bins=feature_size,
add_delta_deltas=False,
audio_frame_length=winlen,
audio_frame_step=winstep)
graph = None
if feat_name == 'fbank':
# get session
if feat_name not in _global_sess:
graph = tf.Graph()
#pylint: disable=not-context-manager
with graph.as_default():
# fbank
filepath = tf.placeholder(dtype=tf.string, shape=[], name='wavpath')
waveforms, sample_rate = speech_ops.read_wav(filepath, params)
del sample_rate
fbank = speech_ops.extract_feature(waveforms, params)
# shape must be [T, D, C]
feat = tf.identity(fbank, name=feat_name)
elif feat_name == 'spec':
# magnitude spec
if feat_name not in _global_sess:
graph = tf.Graph()
#pylint: disable=not-context-manager
with graph.as_default():
filepath = tf.placeholder(dtype=tf.string, shape=[], name='wavpath')
waveforms, sample_rate = speech_ops.read_wav(filepath, params)
spec = py_x_ops.spectrum(
waveforms[:, 0],
tf.cast(sample_rate, tf.dtypes.float32),
output_type=1) #output_type: 1, power spec; 2 log power spec
spec = tf.sqrt(spec)
# shape must be [T, D, C]
spec = tf.expand_dims(spec, -1)
feat = tf.identity(spec, name=feat_name)
else:
raise ValueError(f"Not support freq feat: {feat_name}.")
return graph, (_get_out_tensor_name('wavpath',
0), _get_out_tensor_name(feat_name, 0))
def export_inputs(self):
"""Inputs for exported model."""
vocab_dict = load_vocab_dict(self.text_vocab_file_path)
vocab_size = len(vocab_dict)
label_vocab_dict = load_vocab_dict(self.label_vocab_file_paths[0])
label_vocab_size = len(label_vocab_dict)
self.config['data']['vocab_size'] = vocab_size
self.config['data']['label_vocab_size'] = label_vocab_size
input_sentence = tf.placeholder(shape=(None, ),
dtype=tf.string,
name="input_sentence")
input_pipeline_func = self.get_input_pipeline(for_export=True)
token_ids = input_pipeline_func(input_sentence)
token_ids_len = tf.map_fn(
lambda x: compute_sen_lens(x, padding_token=0), token_ids)
export_data = {
"export_inputs": {
"input_sentence": input_sentence
},
"model_inputs": {
"input_enc_x": token_ids,
"input_x_len": token_ids_len
}
}
return export_data
def test_char_cut_tf_str(self):
t_sen_in = tf.placeholder(dtype=tf.string, shape=())
t_sen_out = char_cut_tf(t_sen_in)
with self.cached_session(use_gpu=False, force_gpu=False) as sess:
sen_out = sess.run(t_sen_out, {t_sen_in: "我爱北京天安门"})
logging.info(sen_out.decode("utf-8"))
self.assertEqual("我 爱 北 京 天 安 门", sen_out.decode("utf-8"))
def test_jieba_cut_op_no_file(self):
''' test jieba '''
graph = tf.Graph()
with graph.as_default():
sentence_in = tf.placeholder(dtype=tf.string,
shape=[None],
name="sentence_in")
sentence_out = self.build_op_no_file(sentence_in)
shape_op = tf.shape(sentence_out)
with self.cached_session(use_gpu=False, force_gpu=False) as sess:
# self.assertShapeEqual(tf.shape(sentence_in), tf.shape(sentence_out))
sentence_out_res = test_one(sess, sentence_out,
{sentence_in: ["我爱北京天安门"]})
self.assertEqual("我 爱 北京 天安门",
sentence_out_res[0].decode("utf-8"))
sentence_out_res = test_one(sess, sentence_out,
{sentence_in: ["吉林省长春药店"]})
self.assertEqual("吉林省 长春 药店",
sentence_out_res[0].decode("utf-8"))
sentence_out_res, shape_res = test_one(
sess, [sentence_out, shape_op],
{sentence_in: ["吉林省长春药店", "南京市长江大桥"]})
self.assertEqual(
"吉林省 长春 药店\n南京市 长江大桥", "\n".join([
one_sen.decode("utf-8") for one_sen in sentence_out_res
]))
logging.info(f"shape: {shape_res}")
self.assertAllEqual(shape_res, [2])
def test_char_cut_tf_list(self):
t_sen_in = tf.placeholder(dtype=tf.string, shape=(None, ))
t_sen_out = char_cut_tf(t_sen_in)
with self.cached_session(use_gpu=False, force_gpu=False) as sess:
sen_out = sess.run(t_sen_out, {t_sen_in: ["我爱北京天安门", "天安门前太阳升啊"]})
logging.info([one.decode("utf-8") for one in sen_out])
self.assertAllEqual(["我 爱 北 京 天 安 门", "天 安 门 前 太 阳 升 啊"],
[one.decode("utf-8") for one in sen_out])
def create_serving_input_receiver_fn(self):
''' infer input pipeline '''
# with batch_size
taskconf = self.config['data']['task']
shape = [None] + taskconf['audio']['feature_shape']
logging.debug('serving input shape:{}'.format(shape))
#pylint: disable=no-member
return tf.estimator.export.build_raw_serving_input_receiver_fn(
features={
'inputs':
tf.placeholder(name="inputs", shape=shape, dtype=tf.float32),
'texts':
tf.placeholder(name="texts",
shape=(None, taskconf['text']['max_text_len']),
dtype=tf.int32)
},
default_batch_size=None,
)
def export_inputs(self):
"""Inputs for exported model."""
vocab_dict = load_vocab_dict(self.text_vocab_file_path)
vocab_size = len(vocab_dict)
if self.use_true_length and self.split_token != "":
if self.split_token not in vocab_dict:
raise ValueError(
"The Model uses split token: {}, not in corpus.".format(
self.split_token))
self.config['data']['split_token'] = int(
vocab_dict[self.split_token])
self.config['data']['vocab_size'] = vocab_size
input_sentence = tf.placeholder(shape=(None, ),
dtype=tf.string,
name="input_sentence")
input_pipeline_func = self.get_input_pipeline(for_export=True)
token_ids = input_pipeline_func(input_sentence)
token_ids_len = tf.map_fn(
lambda x: compute_sen_lens(x, padding_token=0), token_ids)
export_data = {
"export_inputs": {
"input_sentence": input_sentence
},
"model_inputs": {
"input_x": token_ids,
"input_x_len": token_ids_len
}
}
if self.use_dense:
input_dense = tf.placeholder(shape=(None, ),
dtype=tf.float32,
name="input_dense")
export_data["export_inputs"]["input_dense"] = input_dense
return export_data
def test_ngram_op_2_order(self):
''' test ngram 2-order op'''
ground_truth_2 = [0, 0, 0, 0, 0, 0, 0]
word_ngram = 2
t_input = tf.placeholder(shape=(4,), dtype=tf.int32)
t_ngram = py_x_ops.ngram(
t_input, word_ngrams=word_ngram, vocab_size=5000, bucket_size=100000)
logging.info("t_ngram: {}".format(t_ngram))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
ngram_result = sess.run(t_ngram, feed_dict={t_input: self.testcase[0]})
self.assertAllEqual(ngram_result, ground_truth_2)
def create_serving_input_receiver_fn(self):
# with batch_size
taskconf = self.config['data']['task']
shape = [None] + taskconf['audio']['feature_shape']
logging.debug('serving input shape:{}'.format(shape))
return tf.estimator.export.build_raw_serving_input_receiver_fn(
features={
'inputs':
tf.placeholder(name="inputs", shape=shape, dtype=tf.float32),
},
default_batch_size=None,
)
def create_serving_input_receiver_fn(self):
# shape must be with batch_size
taskconf = self.config['data']['task']
shape = taskconf['audio']['feature_shape']
shape.insert(0, None)
return tf.estimator.export.build_raw_serving_input_receiver_fn( #pylint:disable=no-member
features={
'inputs':
tf.placeholder(name="inputs", shape=shape, dtype=tf.float32),
},
default_batch_size=None,
)
def test_clean_english_str_tf(self):
t_sentence_in = tf.placeholder(dtype=tf.string)
t_sentence_out = clean_english_str_tf(t_sentence_in)
with self.cached_session(use_gpu=False, force_gpu=False) as sess:
sentence_out = sess.run(t_sentence_out,
{t_sentence_in: "I'd like to have an APPLE! "})
logging.info(sentence_out)
self.assertEqual("i 'd like to have an apple !",
sentence_out.decode("utf-8"))
sentence_out = sess.run(t_sentence_out,
{t_sentence_in: ["I'd like to have an APPLE! "]})
logging.info(sentence_out)
self.assertEqual("i 'd like to have an apple !",
sentence_out[0].decode("utf-8"))
def test_compute_doc_lens(self):
''' compute document length'''
docs = tf.placeholder(dtype=tf.int32)
lens = compute_doc_lens(docs)
with self.cached_session(use_gpu=False, force_gpu=False) as sess:
# test for 1d
res = sess.run(lens, feed_dict={docs: [1, 2, 0, 0]})
self.assertEqual(res, 2)
# test for 2d
res = sess.run(lens,
feed_dict={docs: [[1, 2, 0, 0], [1, 2, 3, 4]]})
self.assertAllEqual(res, [2, 4])
def add_delta_delta(feat, feat_size, order=2):
''' add delta detla '''
feat_name = 'delta_delta'
graph = None
# get session
if feat_name not in _global_sess:
graph = tf.Graph()
#pylint: disable=not-context-manager
with graph.as_default():
fbank = tf.placeholder(
dtype=tf.float32, shape=[None, feat_size, 1], name='fbank')
feat_with_delta_delta = speech_ops.delta_delta(fbank, order=order)
feat_with_delta_delta = tf.identity(feat_with_delta_delta, name=feat_name)
sess = _get_session(feat_name, graph)
feat = sess.run(
_get_out_tensor_name(feat_name, 0), feed_dict={'fbank:0': feat})
return feat
def test_batch_ngram_op_2_order(self):
''' tset batch 2-order ngram '''
ground_truth_2 = [[0, 0, 0, 0, 0, 0, 0], [223, 0, 0, 0, 0, 0, 0],
[0, 8, 5008, 0, 0, 0, 0], [4, 8, 102492, 0, 0, 0, 0],
[0, 0, 10, 5000, 5010, 0, 0],
[2, 5, 3, 103747, 51858, 0, 0],
[7, 2, 1, 24, 50599, 103743, 54395]]
word_ngram = 2
t_input = tf.placeholder(shape=(7, 4), dtype=tf.int32)
t_ngram = py_x_ops.ngram(
t_input, word_ngrams=word_ngram, vocab_size=5000, bucket_size=100000)
logging.info("batch t_ngram: {}".format(t_ngram))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
ngram_result = sess.run(t_ngram, feed_dict={t_input: self.testcase})
ngram_result = [list(res) for res in ngram_result]
self.assertAllEqual(ngram_result, ground_truth_2)
def test_compute_sen_lens(self):
sentences = tf.placeholder(dtype=tf.int32)
lens = compute_sen_lens(sentences)
with self.cached_session(use_gpu=False, force_gpu=False) as sess:
# test for 1d
res = sess.run(lens, feed_dict={sentences: [1, 2, 0, 0]})
self.assertEqual(res, 2)
# test for 2d
res = sess.run(lens,
feed_dict={sentences: [[1, 2, 0, 0], [1, 2, 3, 4]]})
self.assertAllEqual(res, [2, 4])
# test for 3d
res = sess.run(lens,
feed_dict={
sentences: [[[1, 2, 0, 0]], [[1, 2, 3, 4]],
[[1, 0, 0, 0]]]
})
self.assertAllEqual(res, [[2], [4], [1]])
def test_split_one_doc_to_true_len_sens(self):
doc = tf.placeholder(dtype=tf.int32, shape=[None])
split_token = 1
padding_token = 0
max_doc_len = 4
max_sen_len = 5
lens = split_one_doc_to_true_len_sens(doc, split_token, padding_token,
max_doc_len, max_sen_len)
with self.cached_session(use_gpu=False, force_gpu=False) as sess:
res = sess.run(lens,
feed_dict={doc: [2, 3, 1, 2, 1, 2, 3, 4, 5, 6, 1]})
self.assertAllEqual(res, [[2, 3, 0, 0, 0], [2, 0, 0, 0, 0],
[2, 3, 4, 5, 6], [0, 0, 0, 0, 0]])
all_empty = [[0 for _ in range(max_sen_len)]
for _ in range(max_doc_len)]
res = sess.run(lens, feed_dict={doc: []})
self.assertAllEqual(res, all_empty)
res = sess.run(lens, feed_dict={doc: [1, 1, 1, 1, 1]})
self.assertAllEqual(res, all_empty)
def load_wav(wavpath, sr=8000):
'''
audio:
np.float32, shape [None], sample in [-1, 1], using librosa.load
np.int16, shape [None], sample in [-32768, 32767], using scipy.io.wavfile
np.float32, shape[None, audio_channel], sample int [-1, 1], using tf.DecodeWav
return
sr: sample rate
audio: [-1, 1], same to tf.DecodeWav
'''
#from scipy.io import wavfile
#sample_rate, audio = wavfile.read(wavpath)
#samples, sample_rate = librosa.load(wavpath, sr=sr)
feat_name = 'load_wav'
graph = None
# get session
if feat_name not in _global_sess:
graph = tf.Graph()
with graph.as_default():
params = speech_ops.speech_params(sr=sr, audio_desired_samples=-1)
t_wavpath = tf.placeholder(dtype=tf.string, name="wavpath")
t_audio, t_sample_rate = speech_ops.read_wav(t_wavpath, params)
t_audio = tf.identity(t_audio, name="audio")
t_sample_rate = tf.identity(t_sample_rate, name="sample_rate")
sess = _get_session(feat_name, graph)
audio, sample_rate = sess.run([
_get_out_tensor_name('audio', 0),
_get_out_tensor_name('sample_rate', 0)
],
feed_dict={"wavpath:0": wavpath})
audio = audio[:, 0]
assert sample_rate == sr, 'sampling rate must be {}Hz, get {}Hz'.format(
sr, sample_rate)
return sample_rate, audio
def main(_):
if FLAGS.checkpoints:
# Get the checkpoints list from flags and run some basic checks.
checkpoints = [c.strip() for c in FLAGS.checkpoints.split(",")]
checkpoints = [c for c in checkpoints if c]
if not checkpoints:
raise ValueError("No checkpoints provided for averaging.")
if FLAGS.prefix:
checkpoints = [FLAGS.prefix + c for c in checkpoints]
else:
assert FLAGS.num_last_checkpoints >= 1, "Must average at least one model"
assert FLAGS.prefix, ("Prefix must be provided when averaging last"
" N checkpoints")
checkpoint_state = tf.train.get_checkpoint_state(
os.path.dirname(FLAGS.prefix))
# Checkpoints are ordered from oldest to newest.
checkpoints = checkpoint_state.all_model_checkpoint_paths[
-FLAGS.num_last_checkpoints:]
checkpoints = [c for c in checkpoints if checkpoint_exists(c)]
if not checkpoints:
if FLAGS.checkpoints:
raise ValueError("None of the provided checkpoints exist. %s" %
FLAGS.checkpoints)
else:
raise ValueError("Could not find checkpoints at %s" %
os.path.dirname(FLAGS.prefix))
# Read variables from all checkpoints and average them.
logging.info("Reading variables and averaging checkpoints:")
for c in checkpoints:
logging.info("%s ", c)
var_list = tf.train.list_variables(checkpoints[0])
var_values, var_dtypes = {}, {}
for (name, shape) in var_list:
if not name.startswith("global_step"):
var_values[name] = np.zeros(shape)
for checkpoint in checkpoints:
reader = tf.train.load_checkpoint(checkpoint)
for name in var_values:
tensor = reader.get_tensor(name)
var_dtypes[name] = tensor.dtype
var_values[name] += tensor
logging.info("Read from checkpoint %s", checkpoint)
for name in var_values: # Average.
var_values[name] /= len(checkpoints)
with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
tf_vars = [
tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
for v in var_values
]
placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
global_step = tf.Variable(0,
name="global_step",
trainable=False,
dtype=tf.int64)
saver = tf.train.Saver(tf.all_variables())
# Build a model consisting only of variables, set them to the average values.
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for p, assign_op, (name, value) in zip(placeholders, assign_ops,
six.iteritems(var_values)):
sess.run(assign_op, {p: value})
# Use the built saver to save the averaged checkpoint.
saver.save(sess, FLAGS.output_path, global_step=global_step)
logging.info("Averaged checkpoints saved in %s", FLAGS.output_path)