def __init__(self, conf, dataconf, model):
'''Evaluator constructor
Args:
conf: the evaluator configuration as a ConfigParser
dataconf: the database configuration
model: the model to be evaluated
'''
super(DecoderEvaluator, self).__init__(conf, dataconf, model)
#create a decoder object
self.decoder = decoder_factory.factory(
conf.get('decoder', 'decoder'))(conf, model)
def main(_):
'''does everything for testing'''
decoder_cfg_file = None
#read the database config file
parsed_database_cfg = configparser.ConfigParser()
parsed_database_cfg.read(os.path.join(FLAGS.asr_expdir, 'database.cfg'))
database_cfg = dict(parsed_database_cfg.items('database'))
#read the features config file
parsed_feat_cfg = configparser.ConfigParser()
parsed_feat_cfg.read(
os.path.join(FLAGS.asr_expdir, 'model', 'features.cfg'))
feat_cfg = dict(parsed_feat_cfg.items('features'))
#read the asr config file
parsed_asr_cfg = configparser.ConfigParser()
parsed_asr_cfg.read(os.path.join(FLAGS.asr_expdir, 'model', 'asr.cfg'))
asr_cfg = dict(parsed_asr_cfg.items('asr'))
#read the lm config file
parsed_lm_cfg = configparser.ConfigParser()
parsed_lm_cfg.read(os.path.join(FLAGS.lm_expdir, 'model', 'lm.cfg'))
lm_cfg = dict(parsed_lm_cfg.items('lm'))
#read the asr-lm config file
parsed_asr_lm_cfg = configparser.ConfigParser()
parsed_asr_lm_cfg.read('config/asr_lm.cfg')
asr_lm_cfg = dict(parsed_asr_lm_cfg.items('asr-lm'))
#read the decoder config file
if decoder_cfg_file is None:
decoder_cfg_file = os.path.join(FLAGS.asr_expdir, 'model',
'decoder.cfg')
parsed_decoder_cfg = configparser.ConfigParser()
parsed_decoder_cfg.read(decoder_cfg_file)
decoder_cfg = dict(parsed_decoder_cfg.items('decoder'))
#create a feature reader
featdir = os.path.join(database_cfg['test_dir'], feat_cfg['name'])
with open(os.path.join(featdir, 'maxlength'), 'r') as fid:
max_length = int(fid.read())
reader = feature_reader.FeatureReader(
scpfile=os.path.join(featdir, 'feats.scp'),
cmvnfile=os.path.join(featdir, 'cmvn.scp'),
utt2spkfile=os.path.join(featdir, 'utt2spk'),
max_length=max_length)
#read the feature dimension
with open(
os.path.join(database_cfg['train_dir'], feat_cfg['name'],
'dim'),
'r') as fid:
input_dim = int(fid.read())
#create the coder
with open(os.path.join(database_cfg['train_dir'], 'alphabet')) as fid:
alphabet = fid.read().split(' ')
coder = target_coder.TargetCoder(alphabet)
#create the classifier
classifier = asr_lm_classifier.AsrLmClassifier(
conf=asr_lm_cfg,
asr_conf=asr_cfg,
lm_conf=lm_cfg,
output_dim=coder.num_labels)
#create a decoder
graph = tf.Graph()
with graph.as_default():
decoder = decoder_factory.factory(
conf=decoder_cfg,
classifier=classifier,
input_dim=input_dim,
max_input_length=reader.max_length,
coder=coder,
expdir=FLAGS.asr_expdir)
#create the lm saver
varnames = zip(*checkpoint_utils.list_variables(os.path.join(
FLAGS.lm_expdir, 'model', 'network.ckpt')))[0]
variables = [v for v in tf.all_variables()
if v.name.split(':')[0] in varnames]
lm_saver = tf.train.Saver(variables)
#create the asr saver
varnames = zip(*checkpoint_utils.list_variables(os.path.join(
FLAGS.asr_expdir, 'model', 'network.ckpt')))[0]
variables = [v for v in tf.all_variables()
if v.name.split(':')[0] in varnames]
asr_saver = tf.train.Saver(variables)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True #pylint: disable=E1101
config.allow_soft_placement = True
with tf.Session(graph=graph, config=config) as sess:
#load the lm model
lm_saver.restore(
sess, os.path.join(FLAGS.lm_expdir, 'model', 'network.ckpt'))
#load the asr model
asr_saver.restore(
sess, os.path.join(FLAGS.asr_expdir, 'model', 'network.ckpt'))
#decode with te neural net
decoded = decoder.decode(reader, sess)
#the path to the text file
textfile = database_cfg['testtext']
#read all the reference transcriptions
with open(textfile) as fid:
lines = fid.readlines()
references = dict()
for line in lines:
splitline = line.strip().split(' ')
references[splitline[0]] = ' '.join(splitline[1:])
#compute the character error rate
score = decoder.score(decoded, references)
print 'score: %f' % score
def main(_):
'''does everything for testing'''
decoder_cfg_file = 'config/decoder/attention_visualizer.cfg'
#read the database config file
parsed_database_cfg = configparser.ConfigParser()
parsed_database_cfg.read(os.path.join(FLAGS.expdir, 'database.cfg'))
database_cfg = dict(parsed_database_cfg.items('database'))
#read the features config file
parsed_feat_cfg = configparser.ConfigParser()
parsed_feat_cfg.read(os.path.join(FLAGS.expdir, 'model', 'features.cfg'))
feat_cfg = dict(parsed_feat_cfg.items('features'))
#read the asr config file
parsed_nnet_cfg = configparser.ConfigParser()
parsed_nnet_cfg.read(os.path.join(FLAGS.expdir, 'model', 'asr.cfg'))
nnet_cfg = dict(parsed_nnet_cfg.items('asr'))
#read the decoder config file
if decoder_cfg_file is None:
decoder_cfg_file = os.path.join(FLAGS.expdir, 'model', 'decoder.cfg')
parsed_decoder_cfg = configparser.ConfigParser()
parsed_decoder_cfg.read(decoder_cfg_file)
decoder_cfg = dict(parsed_decoder_cfg.items('decoder'))
#create a feature reader
featdir = os.path.join(database_cfg['test_dir'], feat_cfg['name'])
with open(os.path.join(featdir, 'maxlength'), 'r') as fid:
max_length = int(fid.read())
reader = feature_reader.FeatureReader(
scpfile=os.path.join(featdir, 'feats.scp'),
cmvnfile=os.path.join(featdir, 'cmvn.scp'),
utt2spkfile=os.path.join(featdir, 'utt2spk'),
max_length=max_length)
#read the feature dimension
with open(os.path.join(database_cfg['train_dir'], feat_cfg['name'], 'dim'),
'r') as fid:
input_dim = int(fid.read())
#create the coder
with open(os.path.join(database_cfg['train_dir'], 'alphabet')) as fid:
alphabet = fid.read().split(' ')
coder = target_coder.TargetCoder(alphabet)
#create the classifier
classifier = asr_factory.factory(conf=nnet_cfg,
output_dim=coder.num_labels)
#create a decoder
graph = tf.Graph()
with graph.as_default():
decoder = decoder_factory.factory(conf=decoder_cfg,
classifier=classifier,
input_dim=input_dim,
max_input_length=reader.max_length,
coder=coder,
expdir=FLAGS.expdir)
saver = tf.train.Saver(tf.trainable_variables())
config = tf.ConfigProto()
config.gpu_options.allow_growth = True #pylint: disable=E1101
config.allow_soft_placement = True
with tf.Session(graph=graph, config=config) as sess:
#load the model
saver.restore(sess, os.path.join(FLAGS.expdir, 'model',
'network.ckpt'))
#decode with te neural net
decoded = decoder.decode(reader, sess)
#the path to the text file
textfile = os.path.join(database_cfg['test_dir'], 'targets')
#read all the reference transcriptions
with open(textfile) as fid:
lines = fid.readlines()
references = dict()
for line in lines:
splitline = line.strip().split(' ')
references[splitline[0]] = coder.encode(' '.join(splitline[1:]))
#compute the character error rate
score = decoder.score(decoded, references)
print 'score: %f' % score
#write the resulting beams to disk
decodedir = os.path.join(FLAGS.expdir, 'decoded')
if not os.path.isdir(decodedir):
os.makedirs(decodedir)
for utt in decoded:
with open(os.path.join(decodedir, utt), 'w') as fid:
for hypothesis in decoded[utt]:
fid.write('%f\t%s\n' % (hypothesis[0], hypothesis[1]))
def __init__(self, model, conf, dataconf, expdir):
'''Recognizer constructor
Args:
model: the model to be tested
conf: the recognizer configuration as a configparser
modelconf: the model configuration as a configparser
dataconf: the database configuration as a configparser
expdir: the experiments directory
'''
self.conf = dict(conf.items('recognizer'))
self.expdir = expdir
self.model = model
#get the database configurations
input_sections = [
self.conf[i].split(' ') for i in self.model.input_names
]
self.input_dataconfs = []
for sectionset in input_sections:
self.input_dataconfs.append([])
for section in sectionset:
self.input_dataconfs[-1].append(dict(dataconf.items(section)))
#create a decoder
self.decoder = decoder_factory.factory(conf.get('decoder',
'decoder'))(conf,
self.model)
self.batch_size = int(self.conf['batch_size'])
#create the graph
self.graph = tf.Graph()
with self.graph.as_default():
#get the list of filenames fo the validation set
data_queue_elements, self.names = input_pipeline.get_filenames(
self.input_dataconfs)
#compute the number of batches in the validation set
self.numbatches = int(
math.ceil(float(len(data_queue_elements)) / self.batch_size))
#create a queue to hold the filenames
data_queue = tf.train.string_input_producer(
string_tensor=data_queue_elements,
num_epochs=1,
shuffle=False,
seed=None,
capacity=self.batch_size * 2)
#create the input pipeline
inputs, input_seq_length, _ = input_pipeline.input_pipeline(
data_queue=data_queue,
batch_size=self.batch_size,
numbuckets=1,
allow_smaller_final_batch=True,
dataconfs=self.input_dataconfs)
inputs = {
self.model.input_names[i]: d
for i, d in enumerate(inputs)
}
input_seq_length = {
self.model.input_names[i]: d
for i, d in enumerate(input_seq_length)
}
self.decoded = self.decoder(inputs, input_seq_length)
#create a histogram for all trainable parameters
for param in tf.trainable_variables():
tf.summary.histogram(param.name, param)
def main(_):
'''does everything for testing'''
decoder_cfg_file = None
#read the database config file
parsed_database_cfg = configparser.ConfigParser()
parsed_database_cfg.read(os.path.join(FLAGS.expdir, 'database.cfg'))
database_cfg = dict(parsed_database_cfg.items('database'))
#read the asr config file
parsed_nnet_cfg = configparser.ConfigParser()
parsed_nnet_cfg.read(os.path.join(FLAGS.expdir, 'model', 'lm.cfg'))
nnet_cfg = dict(parsed_nnet_cfg.items('lm'))
#read the decoder config file
if decoder_cfg_file is None:
decoder_cfg_file = os.path.join(FLAGS.expdir, 'model', 'decoder.cfg')
parsed_decoder_cfg = configparser.ConfigParser()
parsed_decoder_cfg.read(decoder_cfg_file)
decoder_cfg = dict(parsed_decoder_cfg.items('decoder'))
#create the coder
with open(os.path.join(FLAGS.expdir, 'model', 'alphabet')) as fid:
alphabet = fid.read().split(' ')
coder = target_coder.TargetCoder(alphabet)
#read the maximum length
with open(os.path.join(database_cfg['test_dir'], 'max_num_chars')) as fid:
max_length = int(fid.read())
#create a text reader
textreader = text_reader.TextReader(textfile=os.path.join(
database_cfg['test_dir'], 'text'),
max_length=max_length,
coder=coder)
#create the classifier
classifier = lm_factory.factory(conf=nnet_cfg, output_dim=coder.num_labels)
#create a decoder
graph = tf.Graph()
with graph.as_default():
decoder = decoder_factory.factory(conf=decoder_cfg,
classifier=classifier,
input_dim=1,
max_input_length=max_length,
coder=coder,
expdir=FLAGS.expdir)
saver = tf.train.Saver(tf.trainable_variables())
config = tf.ConfigProto()
config.gpu_options.allow_growth = True #pylint: disable=E1101
config.allow_soft_placement = True
with tf.Session(graph=graph, config=config) as sess:
#load the model
saver.restore(sess, os.path.join(FLAGS.expdir, 'model',
'network.ckpt'))
#decode with te neural net
decoded = decoder.decode(textreader, sess)
#compute the character error rate
score = decoder.score(decoded, None)
print 'perplexity: %f' % score
def main(_):
'''does everything for testing'''
#decoder_cfg_file = 'config/decoder/attention_visualizer.cfg'
decoder_cfg_file = None
#read the database config file
parsed_database_cfg = configparser.ConfigParser()
parsed_database_cfg.read(os.path.join(FLAGS.expdir, 'database.cfg'))
database_cfg = dict(parsed_database_cfg.items('database'))
# check the training mode
if database_cfg['train_mode'] == 'supervised':
nonsupervised = False
elif database_cfg['train_mode'] == 'semisupervised':
nonsupervised = True
elif database_cfg['train_mode'] == 'nonsupervised':
raise Exception('Purely nonsupervised models should be tested with \
the test_reconstruction file.')
else:
raise Exception('Wrong kind of training mode')
#read the features config file
parsed_feat_cfg = configparser.ConfigParser()
parsed_feat_cfg.read(os.path.join(FLAGS.expdir, 'model', 'features.cfg'))
feat_cfg = dict(parsed_feat_cfg.items('features'))
#read the asr config file
parsed_nnet_cfg = configparser.ConfigParser()
parsed_nnet_cfg.read(os.path.join(FLAGS.expdir, 'model', 'asr.cfg'))
nnet_cfg = dict(parsed_nnet_cfg.items('asr'))
# read the trainer config file
parsed_trainer_cfg = configparser.ConfigParser()
parsed_trainer_cfg.read(os.path.join(FLAGS.expdir, 'trainer.cfg'))
trainer_cfg = dict(parsed_trainer_cfg.items('trainer'))
#read the decoder config file
if decoder_cfg_file is None:
decoder_cfg_file = os.path.join(FLAGS.expdir, 'model', 'decoder.cfg')
parsed_decoder_cfg = configparser.ConfigParser()
parsed_decoder_cfg.read(decoder_cfg_file)
decoder_cfg = dict(parsed_decoder_cfg.items('decoder'))
# if (partly) nonsupervised, check what kind reconstruction features used
# for now two options are implemented
if nonsupervised:
if trainer_cfg['reconstruction_features'] == 'audio_samples':
audio_used = True
else:
audio_used = False
if nonsupervised:
if audio_used:
#read the quantization config file if necessary
parsed_quant_cfg = configparser.ConfigParser()
parsed_quant_cfg.read(os.path.join(FLAGS.expdir, 'model',
'quantization.cfg'))
quant_cfg = dict(parsed_quant_cfg.items('features'))
#create a feature reader
featdir = os.path.join(database_cfg['test_dir'], feat_cfg['name'])
with open(os.path.join(featdir, 'maxlength'), 'r') as fid:
max_length = int(fid.read())
reader = feature_reader.FeatureReader(
scpfile=os.path.join(featdir, 'feats.scp'),
cmvnfile=os.path.join(featdir, 'cmvn.scp'),
utt2spkfile=os.path.join(featdir, 'utt2spk'),
max_length=max_length)
#read the feature dimension
with open(
os.path.join(database_cfg['train_dir'], feat_cfg['name'],
'dim'),
'r') as fid:
input_dim = int(fid.read())
#create the coder
with open(os.path.join(database_cfg['train_dir'], 'alphabet')) as fid:
alphabet = fid.read().split(' ')
coder = target_coder.TargetCoder(alphabet)
#create the classifier
if not nonsupervised:
outputdim2 = 1
else:
if audio_used:
outputdim2 = int(quant_cfg['quant_levels'])
else: #then input features used
outputdim2 = input_dim
classifier = asr_factory.factory(
conf=nnet_cfg,
output_dim=(coder.num_labels, outputdim2))
#create a decoder
graph = tf.Graph()
with graph.as_default():
decoder = decoder_factory.factory(
conf=decoder_cfg,
classifier=classifier,
input_dim=input_dim,
max_input_length=reader.max_length,
coder=coder,
expdir=FLAGS.expdir)
saver = tf.train.Saver(tf.trainable_variables())
config = tf.ConfigProto()
config.gpu_options.allow_growth = True #pylint: disable=E1101
config.allow_soft_placement = True
with tf.Session(graph=graph, config=config) as sess:
#load the model
saver.restore(sess, os.path.join(FLAGS.expdir, 'model', 'network.ckpt'))
#decode with te neural net
decoded = decoder.decode(reader, sess)
#the path to the text file
textfile = database_cfg['testtext']
#read all the reference transcriptions
with open(textfile) as fid:
lines = fid.readlines()
references = dict()
for line in lines:
splitline = line.strip().split(' ')
references[splitline[0]] = coder.encode(' '.join(splitline[1:]))
#compute the character error rate
score = decoder.score(decoded, references)
print 'score: %f' % score
#write the resulting beams to disk
decodedir = os.path.join(FLAGS.expdir, 'decoded')
if not os.path.isdir(decodedir):
os.makedirs(decodedir)
for utt in decoded:
with open(os.path.join(decodedir, utt), 'w') as fid:
for hypothesis in decoded[utt]:
fid.write('%f\t%s\n' % (hypothesis[0], hypothesis[1]))