def build_graph(self, DeployModel):
check_dir(self.config.graph_path)
config_path = path_join(self.config.graph_path, 'config.pkl')
graph_path = path_join(self.config.graph_path, self.config.graph_name)
import pickle
pickle.dump(self.config, open(config_path, 'wb'))
with tf.Graph().as_default(), tf.Session(config=tf.ConfigProto(
allow_soft_placement=True)) as session:
with tf.variable_scope("model"):
model = DeployModel(config=config)
print('Graph build finished')
variable_names = [n.name for n in
tf.get_default_graph().as_graph_def().node]
for n in variable_names:
print(n)
saver = tf.train.Saver()
saver.restore(session, save_path=path_join(self.config.model_path,
'latest.ckpt'))
print("model restored from %s" % config.model_path)
frozen_graph_def = graph_util.convert_variables_to_constants(
session, session.graph.as_graph_def(),
['model/inputX', 'model/rnn_initial_states',
'model/rnn_states', 'model/softmax', 'model/logit'])
tf.train.write_graph(
frozen_graph_def,
self.config.graph_path,
self.config.graph_name,
as_text=False,
)
try:
tf.import_graph_def(frozen_graph_def, name="")
except Exception as e:
print("!!!!Import graph meet error: ", e)
exit()
print('graph saved in %s' % graph_path)
print("====octbit graph====")
rewriter = GraphRewriter(frozen_graph_def, mode="octbit")
octbit_graph = rewriter.rewrite(
['model/inputX', 'model/rnn_initial_states',
'model/rnn_states', 'model/softmax', 'model/logit'])
try:
tf.import_graph_def(octbit_graph, name="")
except Exception as e:
print("!!!!Import octbit graph meet error: ", e)
exit()
tf.train.write_graph(
octbit_graph,
self.config.graph_path,
self.config.graph_name[:-3] + '_octbit.pb',
as_text=False,
)
def __init__(self, sess, network, imdb, roidb, valroidb, output_dir, tbdir, pretrained_model=None):
self.net = network
self.imdb = imdb
self.roidb = roidb
self.valroidb = valroidb
self.output_dir = output_dir
self.tbdir = tbdir
# Simply put '_val' at the end to save the summaries from the validation set
self.tbvaldir = tbdir + '_val'
common.check_dir(self.tbvaldir)
self.pretrained_model = pretrained_model
def __init__(self,
sess,
network,
imdb,
roidb,
valroidb,
output_dir,
tbdir,
pretrained_model=None):
self.net = network
self.imdb = imdb
self.roidb = roidb
self.valroidb = valroidb
self.output_dir = output_dir
self.tbdir = tbdir
# Simply put '_val' at the end to save the summaries from the validation set
self.tbvaldir = tbdir + '_val'
common.check_dir(self.tbvaldir)
self.pretrained_model = pretrained_model
return len(y)
import re
dir = re.sub(r'[^//]+.pkl', '', pkl_path)
with open(pkl_path, "rb") as f:
training_data = pickle.load(f)
sorted_data = sorted(training_data,
key=lambda a: get_len(dir + a[0]))
with open(pkl_path + '.sorted', "wb") as f:
pickle.dump(sorted_data, f)
y, sr = librosa.load(dir + sorted_data[-1][0])
print(len(y))
if __name__ == '__main__':
check_dir(save_train_dir)
check_dir(save_valid_dir)
check_dir(save_noise_dir)
base_pkl = 'vad_train.pkl'
# sort_wave(wave_train_dir + base_pkl)
# generate_trainning_data(
# wave_train_dir + base_pkl + '.sorted')
# sort_wave(wave_valid_dir + "vad_valid.pkl")
generate_valid_data(wave_valid_dir + "vad_valid.pkl.sorted")
# generate_noise_data(wave_noise_dir + 'vad_noise.pkl')
def run(self, TrainingModel):
graph = tf.Graph()
with graph.as_default(), tf.Session() as sess:
self.data = read_dataset(self.config)
if config.mode == 'train':
print('building training model....')
with tf.variable_scope("model"):
self.train_model = TrainingModel(self.config,
self.data.batch_input_queue(),
is_train=True)
self.train_model.config.show()
print('building valid model....')
with tf.variable_scope("model", reuse=True):
self.valid_model = TrainingModel(self.config,
self.data.valid_queue(),
is_train=False)
else:
with tf.variable_scope("model", reuse=False):
self.valid_model = TrainingModel(self.config,
self.data.valid_queue(),
is_train=False)
saver = tf.train.Saver()
# restore from stored models
files = glob(path_join(self.config.model_path, '*.ckpt.*'))
if len(files) > 0:
saver.restore(sess, path_join(self.config.model_path,
self.config.model_name))
print(('Model restored from:' + self.config.model_path))
else:
print("Model doesn't exist.\nInitializing........")
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
tf.Graph.finalize(graph)
st_time = time.time()
if os.path.exists(path_join(self.config.model_path, 'best.pkl')):
with open(path_join(self.config.model_path, 'best.pkl'),
'rb') as f:
best_miss, best_false = pickle.load(f)
print('best miss', best_miss, 'best false', best_false)
else:
print('best not exist')
check_dir(self.config.model_path)
if self.config.mode == 'train':
best_miss = 1
best_false = 1
accu_loss = 0
epoch_step = config.tfrecord_size * self.data.train_file_size // config.batch_size
if self.config.reset_global:
sess.run(self.train_model.reset_global_step)
def handler_stop_signals(signum, frame):
global run
run = False
if not DEBUG:
print(
'training shut down, total setp %s, the model will be save in %s' % (
step, self.config.model_path))
saver.save(sess, save_path=(
path_join(self.config.model_path, 'latest.ckpt')))
print('best miss rate:%f\tbest false rate %f' % (
best_miss, best_false))
sys.exit(0)
signal.signal(signal.SIGINT, handler_stop_signals)
signal.signal(signal.SIGTERM, handler_stop_signals)
best_list = []
best_threshold = 0.08
best_count = 0
# (miss,false,step,best_count)
last_time = time.time()
try:
sess.run([self.data.noise_stage_op,
self.data.noise_filequeue_enqueue_op,
self.train_model.stage_op,
self.train_model.input_filequeue_enqueue_op,
self.valid_model.stage_op,
self.valid_model.input_filequeue_enqueue_op])
va = tf.trainable_variables()
for i in va:
print(i.name)
while self.epoch < self.config.max_epoch:
_, _, _, _, _, l, lr, step, grads = sess.run(
[self.train_model.train_op,
self.data.noise_stage_op,
self.data.noise_filequeue_enqueue_op,
self.train_model.stage_op,
self.train_model.input_filequeue_enqueue_op,
self.train_model.loss,
self.train_model.learning_rate,
self.train_model.global_step,
self.train_model.grads
])
epoch = step // epoch_step
accu_loss += l
if epoch > self.epoch:
self.epoch = epoch
print('accumulated loss', accu_loss)
saver.save(sess, save_path=(
path_join(self.config.model_path,
'latest.ckpt')))
print('latest.ckpt save in %s' % (
path_join(self.config.model_path,
'latest.ckpt')))
accu_loss = 0
if step % config.valid_step == 0:
print('epoch time ', (time.time() - last_time) / 60)
last_time = time.time()
miss_count = 0
false_count = 0
target_count = 0
wer = 0
valid_batch = self.data.valid_file_size * config.tfrecord_size // config.batch_size
text = ""
for i in range(valid_batch):
softmax, correctness, labels, _, _ = sess.run(
[self.valid_model.softmax,
self.valid_model.correctness,
self.valid_model.labels,
self.valid_model.stage_op,
self.valid_model.input_filequeue_enqueue_op])
np.set_printoptions(precision=4,
threshold=np.inf,
suppress=True)
decode_output = [ctc_decode(s) for s in softmax]
for i in decode_output:
text += str(i) + '\n'
text += str(labels) + '\n'
text += '=' * 20 + '\n'
result = [ctc_predict(seq, config.label_seqs)
for seq in
decode_output]
miss, target, false_accept = evaluate(
result, correctness.tolist())
miss_count += miss
target_count += target
false_count += false_accept
wer += self.wer_cal.cal_batch_wer(labels,
decode_output).sum()
# print(miss_count, false_count)
with open('./valid.txt', 'w') as f:
f.write(text)
miss_rate = miss_count / target_count
false_accept_rate = false_count / (
self.data.validation_size - target_count)
print('--------------------------------')
print('epoch %d' % self.epoch)
print('training loss:' + str(l))
print('learning rate:', lr, 'global step', step)
print('miss rate:' + str(miss_rate))
print('flase_accept_rate:' + str(false_accept_rate))
print(miss_count, '/', target_count)
print('wer', wer / self.data.validation_size)
if miss_rate + false_accept_rate < best_miss + best_false:
best_miss = miss_rate
best_false = false_accept_rate
saver.save(sess,
save_path=(path_join(
self.config.model_path,
'best.ckpt')))
with open(path_join(
self.config.model_path, 'best.pkl'),
'wb') as f:
best_tuple = (best_miss, best_false)
pickle.dump(best_tuple, f)
if miss_rate + false_accept_rate < best_threshold:
best_count += 1
print('best_count', best_count)
best_list.append((miss_rate,
false_accept_rate, step,
best_count))
saver.save(sess,
save_path=(path_join(
self.config.model_path,
'best' + str(
best_count) + '.ckpt')))
print(
'training finished, total epoch %d, the model will be save in %s' % (
self.epoch, self.config.model_path))
saver.save(sess, save_path=(
path_join(self.config.model_path, 'latest.ckpt')))
print('best miss rate:%f\tbest false rate"%f' % (
best_miss, best_false))
except tf.errors.OutOfRangeError:
print('Done training -- epoch limit reached')
except Exception as e:
print(e)
traceback.print_exc()
finally:
with open('best_list.pkl', 'wb') as f:
pickle.dump(best_list, f)
print('total time:%f hours' % (
(time.time() - st_time) / 3600))
# When done, ask the threads to stop.
else:
with open(
config.rawdata_path + 'valid/' + "ctc_valid.pkl.sorted",
'rb') as f:
pkl = pickle.load(f)
miss_count = 0
false_count = 0
target_count = 0
valid_batch = self.data.valid_file_size * config.tfrecord_size // config.batch_size
for i in range(valid_batch):
# if i > 7:
# break
ind = 14
softmax, ctc_input, correctness, labels, _, _ = sess.run(
[self.valid_model.softmax,
self.valid_model.nn_outputs,
self.valid_model.correctness,
self.valid_model.labels,
self.valid_model.stage_op,
self.valid_model.input_filequeue_enqueue_op])
np.set_printoptions(precision=4,
threshold=np.inf,
suppress=True)
correctness = correctness.tolist()
decode_output = [ctc_decode(s) for s in softmax]
result = [ctc_predict(seq, config.label_seqs)
for seq in
decode_output]
for k, r in enumerate(result):
if r != correctness[k]:
name = pkl[i * config.batch_size + k][0]
print("scp [email protected]:/ssd/keyword_raw/valid/%s ./"%name)
# print(pkl[i * config.batch_size + k])
# print(decode_output[k])
# print(labels[k])
with open('logits.txt', 'w') as f:
f.write(str(ctc_input[k]))
miss, target, false_accept = evaluate(
result, correctness)
miss_count += miss
target_count += target
false_count += false_accept
print('--------------------------------')
print('miss rate: %d/%d' % (miss_count, target_count))
print('flase_accept_rate: %d/%d' % (
false_count, self.data.validation_size - target_count))
def run(self, TrainingModel):
graph = tf.Graph()
with graph.as_default(), tf.Session() as sess:
self.data = read_dataset(self.config)
if config.mode == 'train':
print('building training model....')
with tf.variable_scope("model"):
self.train_model = TrainingModel(self.config,
self.data.batch_input_queue(),
is_train=True)
self.train_model.config.show()
print('building valid model....')
with tf.variable_scope("model", reuse=True):
self.valid_model = TrainingModel(self.config,
self.data.valid_queue(),
is_train=False)
else:
with tf.variable_scope("model", reuse=False):
self.valid_model = TrainingModel(self.config,
self.data.valid_queue(),
is_train=False)
saver = tf.train.Saver()
# restore from stored models
files = glob(path_join(self.config.model_path, '*.ckpt.*'))
if len(files) > 0:
saver.restore(sess, path_join(self.config.model_path,
self.config.model_name))
print(('Model restored from:' + self.config.model_path))
else:
print("Model doesn't exist.\nInitializing........")
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
tf.Graph.finalize(graph)
best_accuracy = 1
accu_loss = 0
st_time = time.time()
epoch_step = config.tfrecord_size * self.data.train_file_size // config.batch_size
if os.path.exists(path_join(self.config.save_path, 'best.pkl')):
with open(path_join(self.config.save_path, 'best.pkl'),
'rb') as f:
best_accuracy = pickle.load(f)
print('best accuracy', best_accuracy)
else:
print('best not exist')
check_dir(self.config.save_path)
if self.config.mode == 'train':
if self.config.reset_global:
sess.run(self.train_model.reset_global_step)
def handler_stop_signals(signum, frame):
global run
run = False
if not DEBUG:
print(
'training shut down, total setp %s, the model will be save in %s' % (
step, self.config.save_path))
saver.save(sess, save_path=(
path_join(self.config.save_path, 'latest.ckpt')))
print('best accuracy', best_accuracy)
sys.exit(0)
signal.signal(signal.SIGINT, handler_stop_signals)
signal.signal(signal.SIGTERM, handler_stop_signals)
best_list = []
best_threshold = 0.1
best_count = 0
# (miss,false,step,best_count)
last_time = time.time()
try:
sess.run([self.data.noise_stage_op,
self.data.noise_filequeue_enqueue_op,
self.train_model.stage_op,
self.train_model.input_filequeue_enqueue_op,
self.valid_model.stage_op,
self.valid_model.input_filequeue_enqueue_op])
va = tf.trainable_variables()
for i in va:
print(i.name)
while self.epoch < self.config.max_epoch:
_, _, _, _, _, l, lr, step, grads, vs = sess.run(
[self.train_model.train_op,
self.data.noise_stage_op,
self.data.noise_filequeue_enqueue_op,
self.train_model.stage_op,
self.train_model.input_filequeue_enqueue_op,
self.train_model.loss,
self.train_model.learning_rate,
self.train_model.global_step,
self.train_model.grads,
self.train_model.vs
])
# print('-'*30)
# for i in vs:
# print(i.sum())
# print('grads',len(grads))
# print('='*30)
# for g in grads:
# print(g.sum())
epoch = step // epoch_step
accu_loss += l
if epoch > self.epoch:
self.epoch = epoch
print('accumulated loss', accu_loss)
saver.save(sess, save_path=(
path_join(self.config.save_path,
'latest.ckpt')))
print('latest.ckpt save in %s' % (
path_join(self.config.save_path,
'latest.ckpt')))
accu_loss = 0
if step % config.valid_step == 2:
print('epoch time ', (time.time() - last_time) / 60)
last_time = time.time()
total_speech = 0
total_silence = 0
total_miss = 0
total_false = 0
valid_batch = self.data.valid_file_size * config.tfrecord_size // config.batch_size
text = ""
for i in range(valid_batch):
soft, labels, seqlen, _, _ = sess.run(
[
self.valid_model.softmax,
self.valid_model.labels,
self.valid_model.seqLengths,
self.valid_model.stage_op,
self.valid_model.input_filequeue_enqueue_op])
np.set_printoptions(precision=4,
threshold=np.inf,
suppress=True)
# print('-----------------')
#
# print(labels[0])
# print(names[0].decode())
# print(self.valid_set[i * config.batch_size])
# for i in names:
# print(i.decode())
# print(softmax.shape)
# for i, j in zip(soft, labels):
# print(np.concatenate((i, j), 1))
#
# for i in range(len(soft)):
# soft[i][seqlen[i]:] = 0
logits = posterior_predict(soft, config.thres)
target_speech, target_silence, miss, false_trigger = frame_accurcacy(
logits, labels, seqlen)
total_speech += target_speech
total_silence += target_silence
total_miss += miss
total_false += false_trigger
miss_rate = round(total_miss / total_speech, 4)
false_rate = round(total_false / total_silence, 4)
print('--------------------------------')
print('epoch %d' % self.epoch)
print('training loss:' + str(l))
print('learning rate:', lr, 'global step', step)
print('miss rate:' + str(miss_rate))
print('false rate: ' + str(false_rate))
if miss_rate + false_trigger < best_accuracy:
best_accuracy = miss_rate + false_trigger
saver.save(sess,
save_path=(path_join(
self.config.save_path,
'best.ckpt')))
with open(path_join(
self.config.save_path, 'best.pkl'),
'wb') as f:
best_tuple = (miss_rate, false_rate)
pickle.dump(best_tuple, f)
if miss_rate + false_trigger < best_threshold:
best_count += 1
print('best_count', best_count)
best_list.append(
(miss_rate, false_trigger, step,
best_count))
saver.save(sess,
save_path=(path_join(
self.config.save_path,
'best' + str(
best_count) + '.ckpt')))
print(
'training finished, total epoch %d, the model will be save in %s' % (
self.epoch, self.config.save_path))
saver.save(sess, save_path=(
path_join(self.config.save_path, 'latest.ckpt')))
print('best accuracy:%f' % (best_accuracy))
except tf.errors.OutOfRangeError:
print('Done training -- epoch limit reached')
except Exception as e:
print(e)
traceback.print_exc()
finally:
with open('best_list.pkl', 'wb') as f:
pickle.dump(best_list, f)
print('total time:%f hours' % (
(time.time() - st_time) / 3600))
# When done, ask the threads to stop.
else:
total = 0
wrong = 0
valid_batch = self.data.valid_file_size * config.tfrecord_size // config.batch_size
for i in range(valid_batch):
# if i > 7:
# break
ind = 14
logits, labels, _, _ = sess.run(
[self.valid_model.outputs,
self.valid_model.labels,
self.valid_model.stage_op,
self.valid_model.input_filequeue_enqueue_op])
np.set_printoptions(precision=4,
threshold=np.inf,
suppress=True)
total_count, wrong_count = frame_accurcacy(
logits, labels)
total += total_count
wrong += wrong_count
accruracy = 1 - wrong / wrong
# miss_rate = miss_count / target_count
# false_accept_rate = false_count / total_count
print('--------------------------------')
print('accurcay: %f' % (accruracy))
import pickle
from utils.common import path_join, check_dir
from utils.signal import compute_db
import librosa
import random
db_decay = -5
noise_file = '/ssd/keyword/noise/babble.wav'
from config.dnn_config import get_config
config = get_config()
wave_valid_dir = config.rawdata_path + 'train/'
save_valid_dir = config.rawdata_path + 'valid/vad/'
check_dir(save_valid_dir)
def generate(path):
noise, _ = librosa.load(noise_file)
with open(path, 'rb') as f:
wav_list = pickle.load(f)
print('read pkl from %s' % f)
# each record should be (file.wav,((st,end),(st,end).....)))
file_list = [i[0] for i in wav_list]
for f in file_list:
y, _ = librosa.load(path_join(wave_valid_dir, f), sr=config.samplerate)
wave_db = compute_db(y)
start = random.randint(0, len(noise) - len(y))
temp = noise[start:start + len(y)]