def test_context_memory(config_filename, search_engine, src_sentence):
config_training = train_config.load_config_train(config_filename)
(encdec, eos_idx, src_indexer, tgt_indexer
), model_infos = train.create_encdec_and_indexers_from_config_dict(
config_training, load_config_model="yes", return_model_infos=True)
ctxt_mem = create_context_memory(encdec, (src_indexer, tgt_indexer),
search_engine, src_sentence)
def create_and_load_encdec_from_files(config_training_fn, trained_model):
log.info("loading model config from %s" % config_training_fn)
config_training = train_config.load_config_train(config_training_fn)
encdec, eos_idx, src_indexer, tgt_indexer = train.create_encdec_and_indexers_from_config_dict(config_training)
log.info("loading model from %s" % trained_model)
serializers.load_npz(trained_model, encdec)
return encdec, eos_idx, src_indexer, tgt_indexer
def get_src_tgt_dev_from_config_eval(config_eval):
if config_eval.training_config is not None:
training_config_file = config_eval.training_config
elif 'load_model_config' in config_eval.process and config_eval.process.load_model_config is not None:
training_config_file = config_eval.process.load_model_config[0]
log.info("attempting to retrieve dev/test files from %s", training_config_file)
training_config = train_config.load_config_train(training_config_file)
data_prefix = training_config.training_management.data_prefix
data_config_filename = data_prefix + ".data.config"
data_config = make_data_conf.load_config(data_config_filename)
return (data_config["data"]["dev_src"],
data_config["data"]["dev_tgt"],
data_config["data"]["test_src"],
data_config["data"]["test_tgt"])
def create_encdec(config_eval):
encdec_list = []
eos_idx, src_indexer, tgt_indexer = None, None, None
model_infos_list = []
if config_eval.training_config is not None:
assert config_eval.trained_model is not None
encdec, eos_idx, src_indexer, tgt_indexer = create_and_load_encdec_from_files(
config_eval.training_config, config_eval.trained_model)
model_infos_list.append(
create_filename_infos(config_eval.trained_model))
encdec_list.append(encdec)
if 'load_model_config' in config_eval.process and config_eval.process.load_model_config is not None:
for config_filename_and_others in config_eval.process.load_model_config:
other_models_for_averaging = None
if "," in config_filename_and_others:
config_filename_and_others_splitted = config_filename_and_others.split(
",")
config_filename = config_filename_and_others_splitted[0]
other_models_for_averaging = config_filename_and_others_splitted[
1:]
else:
config_filename = config_filename_and_others
log.info("loading model and parameters from config %s" %
config_filename)
config_training = train_config.load_config_train(config_filename)
(
encdec, this_eos_idx, this_src_indexer, this_tgt_indexer
), model_infos = train.create_encdec_and_indexers_from_config_dict(
config_training,
load_config_model="yes",
return_model_infos=True,
additional_models_parameters_for_averaging=
other_models_for_averaging)
model_infos_list.append(model_infos)
if eos_idx is None:
assert len(encdec_list) == 0
assert src_indexer is None
assert tgt_indexer is None
eos_idx, src_indexer, tgt_indexer = this_eos_idx, this_src_indexer, this_tgt_indexer
else:
check_if_vocabulary_info_compatible(this_eos_idx,
this_src_indexer,
this_tgt_indexer, eos_idx,
src_indexer, tgt_indexer)
encdec_list.append(encdec)
assert len(encdec_list) > 0
if 'additional_training_config' in config_eval.process and config_eval.process.additional_training_config is not None:
assert len(config_eval.process.additional_training_config) == len(
config_eval.process.additional_trained_model)
for (config_training_fn, trained_model_fn) in six.moves.zip(
config_eval.process.additional_training_config,
config_eval.process.additional_trained_model):
this_encdec, this_eos_idx, this_src_indexer, this_tgt_indexer = create_and_load_encdec_from_files(
config_training_fn, trained_model_fn)
check_if_vocabulary_info_compatible(this_eos_idx, this_src_indexer,
this_tgt_indexer, eos_idx,
src_indexer, tgt_indexer)
model_infos_list.append(create_filename_infos(trained_model_fn))
encdec_list.append(this_encdec)
if config_eval.process.use_chainerx:
if 'gpu' in config_eval.process and config_eval.process.gpu is not None:
encdec_list = [
encdec.to_device("cuda:%i" % config_eval.process.gpu)
for encdec in encdec_list
]
else:
encdec_list = [
encdec.to_device("native:0") for encdec in encdec_list
]
else:
if 'gpu' in config_eval.process and config_eval.process.gpu is not None:
encdec_list = [
encdec.to_gpu(config_eval.process.gpu)
for encdec in encdec_list
]
if 'reverse_training_config' in config_eval.process and config_eval.process.reverse_training_config is not None:
reverse_encdec, reverse_eos_idx, reverse_src_indexer, reverse_tgt_indexer = create_and_load_encdec_from_files(
config_eval.process.reverse_training_config,
config_eval.process.reverse_trained_model)
if eos_idx != reverse_eos_idx:
raise Exception("incompatible models")
if len(src_indexer) != len(reverse_src_indexer):
raise Exception("incompatible models")
if len(tgt_indexer) != len(reverse_tgt_indexer):
raise Exception("incompatible models")
if config_eval.process.gpu is not None:
reverse_encdec = reverse_encdec.to_gpu(config_eval.process.gpu)
else:
reverse_encdec = None
return encdec_list, eos_idx, src_indexer, tgt_indexer, reverse_encdec, model_infos_list
def process_eval_config(config_fn, dest_dir):
log.info("processing eval config %s " % config_fn)
assert config_fn.endswith(eval_config_suffix)
eval_prefix = config_fn[:-len(eval_config_suffix)]
log.info("eval prefix: %s " % eval_prefix)
urlname = filenamize(eval_prefix) + ".html"
log.info("urlname: %s " % eval_prefix)
time_config_created = os.path.getmtime(config_fn)
dest_filename = os.path.join(dest_dir, urlname)
log.info("create file: %s" % dest_filename)
f = open(dest_filename, "w")
config = load_config_eval(config_fn)
# config = json.load(open(config_fn))
# ref_fn = config["ref"]
ref_fn = config.output.ref
if ref_fn is None and "translation_infos" in config:
ref_fn = config["translation_infos"].ref
src_fn = config.process.src_fn
if src_fn is None and "translation_infos" in config:
src_fn = config["translation_infos"].src
training_config_fn = config.training_config
if training_config_fn is None and config.process.load_model_config is not None and len(
config.process.load_model_config) > 0:
training_config_fn = config.process.load_model_config[0]
if "," in training_config_fn:
training_config_fn = training_config_fn.split(",")[0]
config_training = load_config_train(training_config_fn, no_error=True)
description_training = config_training.get("training_management",
{}).get("description", "")
nb_models_used = 0
if config.training_config is not None:
nb_models_used += 1
if "load_model_config" in config.process and config.process.load_model_config is not None:
nb_models_used += len(config.process.load_model_config)
if config.process.additional_training_config is not None:
nb_models_used += len(config.process.additional_training_config)
if ref_fn is not None:
bc_with_unk = bleu_computer.get_bc_from_files(ref_fn, eval_prefix)
bleu_with_unk = bc_with_unk.bleu()
if not os.path.isfile(eval_prefix + ".unk_replaced"):
bc_unk_replaced = "#No unk replaced file#"
bleu_unk_replaced = -1
else:
bc_unk_replaced = bleu_computer.get_bc_from_files(
ref_fn, eval_prefix + ".unk_replaced")
bleu_unk_replaced = bc_unk_replaced.bleu()
else:
bc_with_unk = "#No Ref#"
bc_unk_replaced = "#No Ref#"
bleu_unk_replaced = -1
bleu_with_unk = -1
desc = {
"description_training": description_training,
"nb_models_used": nb_models_used,
"bleu": bleu_unk_replaced if bleu_unk_replaced >= 0 else bleu_with_unk
}
f.write("""<html> <meta charset="UTF-8"> <style>
table, th, td {
border: 1px solid black;
}
</style><body>""")
f.write("Config Filename: %s <p>" % config_fn)
f.write("eval prefix: %s <p>" % eval_prefix)
f.write("config file created/modified:%s <p>" %
time.ctime(time_config_created))
f.write("BLEU: %s <p>" % bc_with_unk)
f.write("BLEU with unk replaced: %s <p>" % bc_unk_replaced)
f.write("<pre><code>\n")
for line in open(config_fn):
f.write(line)
f.write("</code></pre>\n")
lst_outputs = []
if src_fn is not None:
lst_outputs.append(("src", codecs.open(src_fn, encoding="utf8")))
if ref_fn is not None:
lst_outputs.append(("ref", codecs.open(ref_fn, encoding="utf8")))
if os.path.isfile(eval_prefix + ".unprocessed"):
lst_outputs.append(("unprocessed",
codecs.open(eval_prefix + ".unprocessed",
encoding="utf8")))
lst_outputs.append(("out", codecs.open(eval_prefix, encoding="utf8")))
if os.path.isfile(eval_prefix + ".unk_replaced"):
lst_outputs.append(("unk_repl",
codecs.open(eval_prefix + ".unk_replaced",
encoding="utf8")))
tags_list, files_list = zip(*lst_outputs)
for num_line, line_list in enumerate(itertools.izip(*files_list)):
f.write("""<table style="width:100%">""")
f.write("<tr><td>%s</td><td>%i</td></tr>\n" % ("NUM", num_line))
for tag, line in zip(tags_list, line_list):
f.write("<tr><td>%s</td><td>%s</td></tr>\n" %
(tag, cgi.escape(line.encode("utf8"))))
f.write("</table>")
f.write("</body></html>")
return urlname, desc
def process_train_config(config_fn, dest_dir):
log.info("processing train config %s " % config_fn)
assert config_fn.endswith(train_config_suffix)
train_prefix = config_fn[:-len(train_config_suffix)]
log.info("train prefix: %s " % train_prefix)
urlname = filenamize(train_prefix) + ".html"
log.info("urlname: %s " % train_prefix)
dest_filename = os.path.join(dest_dir, urlname)
if os.path.exists(dest_filename):
time_file_modif = os.path.getmtime(dest_filename)
else:
time_file_modif = None
try:
time_last_exp = os.path.getmtime(train_prefix + ".result.sqlite")
except OSError:
time_last_exp = 1
time_config_created = os.path.getmtime(config_fn)
if time_file_modif is not None and (time_last_exp + 20) < time_file_modif:
log.info("skipping creation for %s" % train_prefix)
log.info("create file: %s" % dest_filename)
f = open(dest_filename, "w")
try:
db = sqlite3.connect(train_prefix + ".result.sqlite")
c = db.cursor()
c.execute(
"SELECT date, bleu_info, iteration, loss, bleu, dev_loss, dev_bleu, avg_time, avg_training_loss FROM exp_data"
)
date, bleu_info, iteration, test_loss, test_bleu, dev_loss, dev_bleu, avg_time, avg_training_loss = zip(
*list(c.fetchall()))
infos = dict(last_nb_iterations=iteration[-1],
nb_records=len(test_loss),
date_last_exp=date[-1],
best_test_loss=min(test_loss),
best_dev_loss=min(dev_loss),
max_test_bleu=max(test_bleu * 100),
max_dev_bleu=max(dev_bleu * 100),
best_avg_training_loss=min(avg_training_loss[1:])
if len(avg_training_loss) > 1 else 0,
avg_time_all=sum(t for t in avg_time if t is not None) /
len(avg_time))
except sqlite3.OperationalError:
infos = None
config = load_config_train(config_fn, no_error=True)
# config = json.load(open(config_fn))
# data_prefix = config["command_line"]["data_prefix"]
data_prefix = config.training_management.data_prefix
try:
graph_training.generate_graph(
sqldb=train_prefix + ".result.sqlite",
# lib = "plotly",
dest=os.path.join(dest_dir,
filenamize(train_prefix) + ".graph.html"),
title=train_prefix)
has_graph = True
except sqlite3.OperationalError as e:
log.warn("could not create graph for %s" % train_prefix)
print e
has_graph = False
f.write("<html><body>")
f.write("Config Filename: %s <p>" % config_fn)
f.write("train prefix: %s <p>" % train_prefix)
f.write("data: <a href = '../data/%s'>%s</a><p>" %
(filenamize(data_prefix) + ".html", filenamize(data_prefix)))
if has_graph:
f.write("<a href = '%s'>graph</a><p>" %
(filenamize(train_prefix) + ".graph.html"))
f.write("config file created/modified:%s <p>" %
time.ctime(time_config_created))
f.write("last exp:%s <p>" % time.ctime(time_last_exp))
time_file_modif_new = os.path.getmtime(dest_filename)
f.write("this file created/modified:%s <p>" %
time.ctime(time_file_modif_new))
if infos is not None:
for key in sorted(infos.keys()):
f.write("%s : %r <p>" % (key, infos[key]))
f.write("<pre><code>")
for line in open(config_fn):
f.write(line)
f.write("</code></pre>")
f.write("</body></html>")
description = config.training_management.get("description", "")
return urlname, data_prefix, time_last_exp, infos, description
def create_encdec(config_eval):
encdec_list = []
eos_idx, src_indexer, tgt_indexer = None, None, None
model_infos_list = []
if config_eval.training_config is not None:
assert config_eval.trained_model is not None
encdec, eos_idx, src_indexer, tgt_indexer = create_and_load_encdec_from_files(
config_eval.training_config, config_eval.trained_model)
model_infos_list.append(create_filename_infos(config_eval.trained_model))
encdec_list.append(encdec)
if 'load_model_config' in config_eval.process and config_eval.process.load_model_config is not None:
for config_filename in config_eval.process.load_model_config:
log.info(
"loading model and parameters from config %s" %
config_filename)
config_training = train_config.load_config_train(config_filename)
(encdec, this_eos_idx, this_src_indexer, this_tgt_indexer), model_infos = train.create_encdec_and_indexers_from_config_dict(config_training,
load_config_model="yes",
return_model_infos=True)
model_infos_list.append(model_infos)
if eos_idx is None:
assert len(encdec_list) == 0
assert src_indexer is None
assert tgt_indexer is None
eos_idx, src_indexer, tgt_indexer = this_eos_idx, this_src_indexer, this_tgt_indexer
else:
check_if_vocabulary_info_compatible(this_eos_idx, this_src_indexer, this_tgt_indexer, eos_idx, src_indexer, tgt_indexer)
encdec_list.append(encdec)
assert len(encdec_list) > 0
if 'additional_training_config' in config_eval.process and config_eval.process.additional_training_config is not None:
assert len(config_eval.process.additional_training_config) == len(config_eval.process.additional_trained_model)
for (config_training_fn, trained_model_fn) in zip(config_eval.process.additional_training_config,
config_eval.process.additional_trained_model):
this_encdec, this_eos_idx, this_src_indexer, this_tgt_indexer = create_and_load_encdec_from_files(
config_training_fn, trained_model_fn)
check_if_vocabulary_info_compatible(this_eos_idx, this_src_indexer, this_tgt_indexer, eos_idx, src_indexer, tgt_indexer)
model_infos_list.append(create_filename_infos(trained_model_fn))
# if args.gpu is not None:
# this_encdec = this_encdec.to_gpu(args.gpu)
encdec_list.append(this_encdec)
if 'gpu' in config_eval.process and config_eval.process.gpu is not None:
encdec_list = [encdec.to_gpu(config_eval.process.gpu) for encdec in encdec_list]
if 'reverse_training_config' in config_eval.process and config_eval.process.reverse_training_config is not None:
reverse_encdec, reverse_eos_idx, reverse_src_indexer, reverse_tgt_indexer = create_and_load_encdec_from_files(
config_eval.process.reverse_training_config, config_eval.process.reverse_trained_model)
if eos_idx != reverse_eos_idx:
raise Exception("incompatible models")
if len(src_indexer) != len(reverse_src_indexer):
raise Exception("incompatible models")
if len(tgt_indexer) != len(reverse_tgt_indexer):
raise Exception("incompatible models")
if config_eval.process.gpu is not None:
reverse_encdec = reverse_encdec.to_gpu(config_eval.process.gpu)
else:
reverse_encdec = None
return encdec_list, eos_idx, src_indexer, tgt_indexer, reverse_encdec, model_infos_list