def recognize(self):
'''perform the recognition'''
with self.graph.as_default():
#create a hook that will load the model
load_hook = LoadAtBegin(
os.path.join(self.expdir, 'model', 'network.ckpt'),
self.model.variables)
#create a hook for summary writing
summary_hook = SummaryHook(os.path.join(self.expdir, 'logdir'))
directory = os.path.join(self.expdir, 'decoded')
if os.path.isdir(directory):
shutil.rmtree(directory)
os.makedirs(directory)
#start the session
with tf.train.SingularMonitoredSession(
hooks=[load_hook, summary_hook]) as sess:
nameid = 0
for _ in range(self.numbatches):
#decode
outputs = sess.run(self.decoded)
#write to disk
names = self.names[nameid:nameid + self.batch_size]
#cut of the added index to the name
names = ['-'.join(name.split('-')[:-1]) for name in names]
self.decoder.write(outputs, directory, names)
nameid += self.batch_size
def test(expdir):
'''does everything for testing'''
#read the database config file
database_cfg = configparser.ConfigParser()
database_cfg.read(os.path.join(expdir, 'database.cfg'))
#read the model config file
model_cfg = configparser.ConfigParser()
model_cfg.read(os.path.join(expdir, 'model.cfg'))
#read the evaluator config file
evaluator_cfg = configparser.ConfigParser()
evaluator_cfg.read(os.path.join(expdir, 'evaluator.cfg'))
#quick fix
#evaluator_cfg.set('evaluator','batch_size','5')
#read the reconstructor config file
reconstructor_cfg = configparser.ConfigParser()
reconstructor_cfg.read(os.path.join(expdir, 'reconstructor.cfg'))
#read the scorer config file
scorer_cfg = configparser.ConfigParser()
scorer_cfg.read(os.path.join(expdir, 'scorer.cfg'))
#read the postprocessor config file, if it exists
try:
postprocessor_cfg = configparser.ConfigParser()
postprocessor_cfg.read(os.path.join(expdir, 'postprocessor.cfg'))
if not postprocessor_cfg.sections():
postprocessor_cfg = None
except:
postprocessor_cfg = None
postprocessor_cfg = None
if evaluator_cfg.get('evaluator', 'evaluator') == 'multi_task':
tasks = evaluator_cfg.get('evaluator', 'tasks').split(' ')
else:
raise 'unkown type of evaluation %s' % evaluator_cfg.get(
'evaluator', 'evaluator')
#evaluate each task separately
for task in tasks:
rec_dir = os.path.join(expdir, 'reconstructions', task)
#load the model
with open(os.path.join(expdir, 'model', 'model.pkl'), 'rb') as fid:
models = pickle.load(fid)
if os.path.isfile(os.path.join(expdir, 'loss_%s' % task)):
print 'already reconstructed all signals for task %s, going straight to scoring' % task
if evaluator_cfg.has_option(task, 'requested_utts'):
requested_utts = int(evaluator_cfg.get(task, 'requested_utts'))
else:
requested_utts = int(
evaluator_cfg.get('evaluator', 'requested_utts'))
if evaluator_cfg.has_option(task, 'batch_size'):
batch_size = int(evaluator_cfg.get(task, 'batch_size'))
else:
batch_size = int(evaluator_cfg.get('evaluator', 'batch_size'))
numbatches = int(float(requested_utts) / float(batch_size))
else:
print 'Evaluating task %s' % task
#create the evaluator
evaltype = evaluator_cfg.get(task, 'evaluator')
evaluator = evaluator_factory.factory(evaltype)(
conf=evaluator_cfg,
dataconf=database_cfg,
models=models,
task=task)
#create the reconstructor
task_reconstructor_cfg = dict(reconstructor_cfg.items(task))
reconstruct_type = task_reconstructor_cfg['reconstruct_type']
reconstructor = reconstructor_factory.factory(reconstruct_type)(
conf=task_reconstructor_cfg,
evalconf=evaluator_cfg,
dataconf=database_cfg,
rec_dir=rec_dir,
task=task)
#create the graph
graph = tf.Graph()
with graph.as_default():
#compute the loss
batch_loss, batch_norm, numbatches, batch_outputs, batch_seq_length = evaluator.evaluate(
)
#create a hook that will load the model
load_hook = LoadAtBegin(
os.path.join(expdir, 'model', 'network.ckpt'), models)
#create a hook for summary writing
summary_hook = SummaryHook(os.path.join(expdir, 'logdir'))
config = tf.ConfigProto(device_count={'CPU': 1, 'GPU': 0})
options = tf.RunOptions()
options.report_tensor_allocations_upon_oom = True
#start the session
with tf.train.SingularMonitoredSession(
hooks=[load_hook,
summary_hook], config=config) as sess:
loss = 0.0
loss_norm = 0.0
for batch_ind in range(0, numbatches):
print 'evaluating batch number %d' % batch_ind
last_time = time.time()
[
batch_loss_eval, batch_norm_eval,
batch_outputs_eval, batch_seq_length_eval
] = sess.run(fetches=[
batch_loss, batch_norm, batch_outputs,
batch_seq_length
],
options=options)
loss += batch_loss_eval
loss_norm += batch_norm_eval
print '%f' % (time.time() - last_time)
last_time = time.time()
#chosing the first seq_length
reconstructor(batch_outputs_eval,
batch_seq_length_eval)
print '%f' % (time.time() - last_time)
loss = loss / loss_norm
print 'task %s: loss = %0.6g' % (task, loss)
#write the loss to disk
with open(os.path.join(expdir, 'loss_%s' % task), 'w') as fid:
fid.write(str(loss))
#from here on there is no need for a GPU anymore ==> score script to be run separately on
#different machine?
task_scorer_cfg = dict(scorer_cfg.items(task))
score_types = task_scorer_cfg['score_type'].split(' ')
for score_type in score_types:
if os.path.isfile(
os.path.join(
expdir,
'results_%s_%s_complete.json' % (task, score_type))):
print 'Already found a score for task %s for score type %s, skipping it.' % (
task, score_type)
else:
print 'Scoring task %s for score type %s' % (task, score_type)
#create the scorer
scorer = scorer_factory.factory(score_type)(
conf=task_scorer_cfg,
evalconf=evaluator_cfg,
dataconf=database_cfg,
rec_dir=rec_dir,
numbatches=numbatches,
task=task)
#run the scorer
scorer()
with open(
os.path.join(
expdir, 'results_%s_%s_complete.json' %
(task, score_type)), 'w') as fid:
json.dump(scorer.results, fid)
result_summary = scorer.summarize()
with open(
os.path.join(
expdir,
'results_%s_%s_summary.json' % (task, score_type)),
'w') as fid:
json.dump(result_summary, fid)
if postprocessor_cfg != None: # && postprocessing is not done yet for this task
task_postprocessor_cfg = dict(postprocessor_cfg.items(task))
task_processor_cfg = dict(
postprocessor_cfg.items('processor_' + task))
postprocess_types = task_postprocessor_cfg[
'postprocess_type'].split(' ')
for postprocess_type in postprocess_types:
#create the postprocessor
postprocessor = postprocessor_factory.factory(
postprocess_type)(conf=task_postprocessor_cfg,
proc_conf=task_processor_cfg,
evalconf=evaluator_cfg,
expdir=expdir,
rec_dir=rec_dir,
task=task)
#run the postprocessor
postprocessor()
postprocessor.matlab_eng.quit()
def test(expdir):
'''does everything for testing'''
#read the database config file
database_cfg = configparser.ConfigParser()
database_cfg.read(os.path.join(expdir, 'database.cfg'))
#load the model
with open(os.path.join(expdir, 'model', 'model.pkl'), 'rb') as fid:
model = pickle.load(fid)
#read the evaluator config file
evaluator_cfg = configparser.ConfigParser()
evaluator_cfg.read(os.path.join(expdir, 'evaluator.cfg'))
#create the evaluator
evaltype = evaluator_cfg.get('evaluator', 'evaluator')
evaluator = evaluator_factory.factory(evaltype)(
conf=evaluator_cfg,
dataconf=database_cfg,
model=model)
#create the reconstructor
reconstruct_type = evaluator_cfg.get('reconstructor', 'reconstruct_type')
reconstructor = reconstructor_factory.factory(reconstruct_type)(
conf=evaluator_cfg,
dataconf=database_cfg,
expdir=expdir)
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
#create the graph
graph = tf.Graph()
with graph.as_default():
#compute the loss
batch_loss, numbatches, batch_outputs, batch_seq_length = evaluator.evaluate()
#create a hook that will load the model
load_hook = LoadAtBegin(
os.path.join(expdir, 'model', 'network.ckpt'),
model)
#create a hook for summary writing
summary_hook = SummaryHook(os.path.join(expdir, 'logdir'))
#start the session
with tf.train.SingularMonitoredSession(
hooks=[load_hook, summary_hook]) as sess:
loss = 0.0
for batch_ind in range(0,numbatches):
print 'evaluating batch number %d' %batch_ind
batch_loss_eval, batch_outputs_eval, batch_seq_length_eval = sess.run(
fetches=[batch_loss, batch_outputs, batch_seq_length])
loss += batch_loss_eval
reconstructor(batch_outputs_eval['outputs'],
batch_seq_length_eval['features'])
loss = loss#/numbatches
print 'loss = %0.6g' % loss
#write the loss to disk
with open(os.path.join(expdir, 'loss'), 'w') as fid:
fid.write(str(loss))
#from here on there is no need for a GPU anymore ==> score script to be run separately on
#different machine? reconstructor.rec_dir has to be known though. can be put in evaluator_cfg
score_type = evaluator_cfg.get('scorer', 'score_type')
for i in range(10):
# Sometime it fails and not sure why. Just retry then. max 10 times
try:
#create the scorer
scorer = scorer_factory.factory(score_type)(
conf=evaluator_cfg,
dataconf=database_cfg,
rec_dir=reconstructor.rec_dir,
numbatches=numbatches)
#run the scorer
scorer()
except Exception:
if i==9:
raise Exception
else:
continue
break
with open(os.path.join(expdir, 'results_complete.json'), 'w') as fid:
json.dump(scorer.results,fid)
result_summary = scorer.summarize()
with open(os.path.join(expdir, 'results_summary.json'), 'w') as fid:
json.dump(result_summary,fid)
def test(expdir, testing=False):
'''does everything for testing
args:
expdir: the experiments directory
testing: if true only the graph will be created for debugging purposes
'''
#read the database config file
database_cfg = configparser.ConfigParser()
database_cfg.read(os.path.join(expdir, 'database.conf'))
if testing:
model_cfg = configparser.ConfigParser()
model_cfg.read(os.path.join(expdir, 'model.cfg'))
trainer_cfg = configparser.ConfigParser()
trainer_cfg.read(os.path.join(expdir, 'trainer.cfg'))
model = Model(conf=model_cfg,
trainlabels=int(trainer_cfg.get('trainer',
'trainlabels')))
else:
#load the model
with open(os.path.join(expdir, 'model', 'model.pkl'), 'rb') as fid:
model = pickle.load(fid)
#read the evaluator config file
evaluator_cfg = configparser.ConfigParser()
evaluator_cfg.read(os.path.join(expdir, 'test_evaluator.cfg'))
#create the evaluator
evaltype = evaluator_cfg.get('evaluator', 'evaluator')
evaluator = evaluator_factory.factory(evaltype)(conf=evaluator_cfg,
dataconf=database_cfg,
model=model)
#create the graph
graph = tf.Graph()
with graph.as_default():
#compute the loss
loss, update_loss, numbatches = evaluator.evaluate()
if testing:
return
#create a histogram for all trainable parameters
for param in model.variables:
tf.summary.histogram(param.name, param)
#create a hook that will load the model
load_hook = LoadAtBegin(os.path.join(expdir, 'model', 'network.ckpt'),
model.variables)
#create a hook for summary writing
summary_hook = SummaryHook(os.path.join(expdir, 'logdir'))
#start the session
with tf.train.SingularMonitoredSession(
hooks=[load_hook, summary_hook]) as sess:
for _ in range(numbatches):
update_loss.run(session=sess)
loss = loss.eval(session=sess)
print 'loss = %f' % loss
#write the result to disk
with open(os.path.join(expdir, 'result'), 'w') as fid:
fid.write(str(loss))