def test_multibatch(self):
Vi, Ei, Hi = 12, 17, 7
enc = EncoderNaive(Vi, Ei, Hi)
raw_seq1 = [2, 5, 0, 3]
raw_seq2 = [2, 5, 4, 3, 0, 0, 1, 11, 3]
raw_seq3 = [2, 5, 4, 3, 0, 11, 3]
raw_seq4 = [5, 3, 0, 0, 1, 11, 3]
src_data = [raw_seq1, raw_seq2, raw_seq3, raw_seq4]
src_batch, src_mask = utils.make_batch_src(src_data)
fb = enc(src_batch, src_mask)
for i in range(len(src_data)):
raw_s = src_data[i]
input_seq = [
Variable(np.array([v], dtype=np.int32)) for v in raw_s
]
fb_naive = enc.naive_call(input_seq, None)
for j in range(len(raw_s)):
print("maxdiff:",
np.max(np.abs(fb.data[i][j] - fb_naive[j].data[0])))
assert np.allclose(fb.data[i][j],
fb_naive[j].data[0],
atol=1e-6)
def test_multibatch(self):
Vi, Ei, Hi = 12, 17, 7
enc = EncoderNaive(Vi, Ei, Hi)
Hi_a, Ha, Ho = 2 * Hi, 19, 23
attn_model = AttentionModuleNaive(Hi_a, Ha, Ho)
raw_seq1 = [2, 5, 0, 3]
raw_seq2 = [2, 5, 4, 3, 0, 0, 1, 11, 3]
raw_seq3 = [2, 5, 4, 3, 0, 11, 3]
raw_seq4 = [5, 3, 0, 0, 1, 11, 3]
src_data = [raw_seq1, raw_seq2, raw_seq3, raw_seq4]
src_batch, src_mask = utils.make_batch_src(src_data)
fb = enc(src_batch, src_mask)
compute_ctxt = attn_model(fb, src_mask)
state_raw = np.random.randn(4, Ho).astype(np.float32)
state = Variable(state_raw)
ci, attn = compute_ctxt(state)
for i in range(len(src_data)):
raw_s = src_data[i]
input_seq = [
Variable(np.array([v], dtype=np.int32)) for v in raw_s
]
fb_naive = enc.naive_call(input_seq, None)
compute_ctxt_naive = attn_model.naive_call(fb_naive, None)
ci_naive, attn_naive = compute_ctxt_naive(
Variable(state_raw[i].reshape(1, -1)))
print("maxdiff ci:", np.max(np.abs(ci.data[i] - ci_naive.data[0])))
assert np.allclose(ci.data[i], ci_naive.data[0], atol=1e-6)
# print(attn.data.shape, attn_naive.data.shape)
print(
"maxdiff attn:",
np.max(np.abs(attn.data[i][:len(raw_s)] - attn_naive.data[0])))
assert np.allclose(attn.data[i][:len(raw_s)],
attn_naive.data[0],
atol=1e-6)
assert np.all(attn.data[i][len(raw_s):] == 0)
def do_eval(config_eval):
src_fn = config_eval.process.src_fn
tgt_fn = config_eval.output.tgt_fn
mode = config_eval.method.mode
gpu = config_eval.process.gpu
dest_fn = config_eval.process.dest_fn
mb_size = config_eval.process.mb_size
nb_steps = config_eval.method.nb_steps
nb_steps_ratio = config_eval.method.nb_steps_ratio
max_nb_ex = config_eval.process.max_nb_ex
nbest_to_rescore = config_eval.output.nbest_to_rescore
nbest = config_eval.output.nbest
beam_width = config_eval.method.beam_width
beam_pruning_margin = config_eval.method.beam_pruning_margin
beam_score_length_normalization = config_eval.method.beam_score_length_normalization
beam_score_length_normalization_strength = config_eval.method.beam_score_length_normalization_strength
beam_score_coverage_penalty = config_eval.beam_score_coverage_penalty
beam_score_coverage_penalty_strength = config_eval.beam_score_coverage_penalty_strength
always_consider_eos_and_placeholders = config_eval.method.always_consider_eos_and_placeholders
if config_eval.process.force_placeholders:
# making it default for now
always_consider_eos_and_placeholders = True
post_score_length_normalization = config_eval.method.post_score_length_normalization
post_score_length_normalization_strength = config_eval.method.post_score_length_normalization_strength
groundhog = config_eval.method.groundhog
tgt_unk_id = config_eval.output.tgt_unk_id
force_finish = config_eval.method.force_finish
prob_space_combination = config_eval.method.prob_space_combination
generate_attention_html = config_eval.output.generate_attention_html
rich_output_filename = config_eval.output.rich_output_filename
ref = config_eval.output.ref
dic = config_eval.output.dic
normalize_unicode_unk = config_eval.output.normalize_unicode_unk
attempt_to_relocate_unk_source = config_eval.output.attempt_to_relocate_unk_source
remove_unk = config_eval.output.remove_unk
post_score_coverage_penalty = config_eval.method.post_score_coverage_penalty
post_score_coverage_penalty_strength = config_eval.method.post_score_coverage_penalty_strength
time_start = time.perf_counter()
astar_params = beam_search.AStarParams(
astar_batch_size=config_eval.method.astar_batch_size,
astar_max_queue_size=config_eval.method.astar_max_queue_size,
astar_prune_margin=config_eval.method.astar_prune_margin,
astar_prune_ratio=config_eval.method.astar_prune_ratio,
length_normalization_exponent=config_eval.method.
astar_length_normalization_exponent,
length_normalization_constant=config_eval.method.
astar_length_normalization_constant,
astar_priority_eval_string=config_eval.method.
astar_priority_eval_string,
max_length_diff=config_eval.method.astar_max_length_diff)
make_constraints_dict = None
if config_eval.process.server is None:
encdec_list, eos_idx, src_indexer, tgt_indexer, reverse_encdec, model_infos_list = create_encdec(
config_eval)
eval_dir_placeholder = "@eval@/"
if dest_fn.startswith(eval_dir_placeholder):
if config_eval.trained_model is not None:
training_model_filename = config_eval.trained_model
else:
if len(config_eval.process.load_model_config) == 0:
log.error("Cannot detect value for $eval$ placeholder")
sys.exit(1)
training_model_filename = config_eval.process.load_model_config[
0]
eval_dir = os.path.join(os.path.dirname(training_model_filename),
"eval")
dest_fn = os.path.join(eval_dir,
dest_fn[len(eval_dir_placeholder):])
log.info("$eval$ detected. dest_fn is: %s ", dest_fn)
ensure_path(eval_dir)
if src_fn is None:
(dev_src_from_config, dev_tgt_from_config, test_src_from_config,
test_tgt_from_config
) = get_src_tgt_dev_from_config_eval(config_eval)
if test_src_from_config is None:
log.error(
"Could not find value for source text, either on command line or in config files"
)
sys.exit(1)
log.info("using files from config as src:%s", test_src_from_config)
src_fn = test_src_from_config
if ref is None:
log.info("using files from config as ref:%s",
test_tgt_from_config)
ref = test_tgt_from_config
if config_eval.process.force_placeholders:
if make_constraints_dict is None:
make_constraints_dict = {}
make_constraints_dict[
"ph_constraint"] = placeholder_constraints_builder(
src_indexer,
tgt_indexer,
units_placeholders=config_eval.process.units_placeholders)
if config_eval.process.bilingual_dic_for_reranking:
if make_constraints_dict is None:
make_constraints_dict = {}
print("**making ja en dic")
ja_en_search, en_ja_search = dictionnary_handling.load_search_trie(
config_eval.process.bilingual_dic_for_reranking,
config_eval.process.invert_bilingual_dic_for_reranking)
print("**define constraints")
make_constraints_dict[
"dic_constraint"] = dictionnary_handling.make_constraint(
ja_en_search, en_ja_search, tgt_indexer)
elif False:
re_word = re.compile(r"[A-Za-z]+")
re_digits = re.compile(r"\d+")
def unsegment(s):
res = []
for w in s.split(" "):
if w.startswith("▁"):
w = " " + w[1:]
res.append(w)
return "".join(res)
def make_constraints(src, src_seq):
line_src = unsegment(src)
line_src = unicodedata.normalize('NFKC', line_src)
word_list = [
word for word in re_word.findall(line_src) if len(word) > 3
]
digit_list = [
digit for digit in re_digits.findall(line_src)
if len(digit) > 2
]
if len(word_list) == 0 and len(digit_list) == 0:
def constraint_fn(tgt_seq):
return 1
else:
def constraint_fn(tgt_seq):
tgt = tgt_indexer.deconvert(tgt_seq)
line_tgt = unsegment(tgt)
line_tgt = unicodedata.normalize('NFKC', line_tgt)
matched_word = 0
for word in word_list:
if word in line_ref:
matched_word += 1
matched_digit = 0
for digit in digit_list:
if digit in line_ref:
matched_digit += 1
if matched_word == len(
word_list) and matched_digit == len(
digit_list):
return 1
else:
return (matched_word + matched_digit) / (
len(word_list) + len(digit_list))
return constraint_fn
else:
make_constraints_dict = None
log.info("opening source file %s" % src_fn)
preprocessed_input = build_dataset_one_side_pp(
src_fn,
src_pp=src_indexer,
max_nb_ex=max_nb_ex,
make_constraints_dict=make_constraints_dict)
if make_constraints_dict is not None:
src_data, stats_src_pp, constraints_list = preprocessed_input
else:
src_data, stats_src_pp = preprocessed_input
constraints_list = None
log.info("src data stats:\n%s", stats_src_pp.make_report())
translation_infos = OrderedNamespace()
translation_infos["src"] = src_fn
translation_infos["tgt"] = tgt_fn
translation_infos["ref"] = ref
for num_model, model_infos in enumerate(model_infos_list):
translation_infos["model%i" % num_model] = model_infos
if dest_fn is not None:
save_eval_config_fn = dest_fn + ".eval.init.config.json"
log.info("Saving initial eval config to %s" % save_eval_config_fn)
config_eval.save_to(save_eval_config_fn)
# log.info("%i sentences loaded" % make_data_infos.nb_ex)
# log.info("#tokens src: %i of which %i (%f%%) are unknown"%(make_data_infos.total_token,
# make_data_infos.total_count_unk,
# float(make_data_infos.total_count_unk * 100) /
# make_data_infos.total_token))
tgt_data = None
if tgt_fn is not None:
log.info("opening target file %s" % tgt_fn)
tgt_data, stats_tgt_pp = build_dataset_one_side_pp(tgt_fn,
src_pp=tgt_indexer,
max_nb_ex=max_nb_ex)
log.info("tgt data stats:\n%s", stats_tgt_pp.make_report())
# log.info("%i sentences loaded"%make_data_infos.nb_ex)
# log.info("#tokens src: %i of which %i (%f%%) are unknown"%(make_data_infos.total_token,
# make_data_infos.total_count_unk,
# float(make_data_infos.total_count_unk * 100) /
# make_data_infos.total_token))
# translations = greedy_batch_translate(encdec, eos_idx, src_data, batch_size = mb_size, gpu = args.gpu)
time_all_loaded = time.perf_counter()
if mode == "translate":
log.info("writing translation of to %s" % dest_fn)
with cuda.get_device_from_id(gpu):
assert len(encdec_list) == 1
translations = greedy_batch_translate(
encdec_list[0],
eos_idx,
src_data,
batch_size=mb_size,
gpu=gpu,
nb_steps=nb_steps,
use_chainerx=config_eval.process.use_chainerx)
out = io.open(dest_fn, "wt", encoding="utf8")
for t in translations:
if t[-1] == eos_idx:
t = t[:-1]
ct = tgt_indexer.deconvert(t, unk_tag="#T_UNK#")
# ct = convert_idx_to_string(t, tgt_voc + ["#T_UNK#"])
out.write(ct + "\n")
elif mode == "beam_search" or mode == "eval_bleu" or mode == "astar_search" or mode == "astar_eval_bleu":
if config_eval.process.server is not None:
from nmt_chainer.translation.server import do_start_server
do_start_server(config_eval)
else:
def translate_closure(beam_width, nb_steps_ratio):
beam_search_params = beam_search.BeamSearchParams(
beam_width=beam_width,
beam_pruning_margin=beam_pruning_margin,
beam_score_coverage_penalty=beam_score_coverage_penalty,
beam_score_coverage_penalty_strength=
beam_score_coverage_penalty_strength,
beam_score_length_normalization=
beam_score_length_normalization,
beam_score_length_normalization_strength=
beam_score_length_normalization_strength,
force_finish=force_finish,
use_unfinished_translation_if_none_found=True,
always_consider_eos_and_placeholders=
always_consider_eos_and_placeholders)
translate_to_file_with_beam_search(
dest_fn,
gpu,
encdec_list,
eos_idx,
src_data,
beam_search_params=beam_search_params,
nb_steps=nb_steps,
nb_steps_ratio=nb_steps_ratio,
post_score_length_normalization=
post_score_length_normalization,
post_score_length_normalization_strength=
post_score_length_normalization_strength,
post_score_coverage_penalty=post_score_coverage_penalty,
post_score_coverage_penalty_strength=
post_score_coverage_penalty_strength,
groundhog=groundhog,
tgt_unk_id=tgt_unk_id,
tgt_indexer=tgt_indexer,
prob_space_combination=prob_space_combination,
reverse_encdec=reverse_encdec,
generate_attention_html=generate_attention_html,
src_indexer=src_indexer,
rich_output_filename=rich_output_filename,
unprocessed_output_filename=dest_fn + ".unprocessed",
nbest=nbest,
constraints_fn_list=constraints_list,
use_astar=(mode == "astar_search"
or mode == "astar_eval_bleu"),
astar_params=astar_params,
use_chainerx=config_eval.process.use_chainerx)
translation_infos["dest"] = dest_fn
translation_infos["unprocessed"] = dest_fn + ".unprocessed"
if mode == "eval_bleu" or mode == "astar_eval_bleu":
if ref is not None:
bc = bleu_computer.get_bc_from_files(ref, dest_fn)
print("bleu before unk replace:", bc)
translation_infos["bleu"] = bc.bleu()
translation_infos["bleu_infos"] = str(bc)
else:
print("bleu before unk replace: No Ref Provided")
from nmt_chainer.utilities import replace_tgt_unk
replace_tgt_unk.replace_unk(
dest_fn, src_fn, dest_fn + ".unk_replaced", dic,
remove_unk, normalize_unicode_unk,
attempt_to_relocate_unk_source)
translation_infos[
"unk_replaced"] = dest_fn + ".unk_replaced"
if ref is not None:
bc = bleu_computer.get_bc_from_files(
ref, dest_fn + ".unk_replaced")
print("bleu after unk replace:", bc)
translation_infos["post_unk_bleu"] = bc.bleu()
translation_infos["post_unk_bleu_infos"] = str(bc)
else:
print("bleu before unk replace: No Ref Provided")
return -bc.bleu()
else:
return None
if config_eval.process.do_hyper_param_search is not None:
study_filename, study_name, n_trials = do_hyper_param_search
n_trials = int(n_trials)
import optuna
def objective(trial):
nb_steps_ratio = trial.suggest_uniform(
'nb_steps_ratio', 0.9, 3.5)
beam_width = trial.suggest_int("beam_width", 2, 50)
return translate_closure(beam_width, nb_steps_ratio)
study = optuna.create_study(study_name=study_name,
storage="sqlite:///" +
study_filename)
study.optimize(objective, n_trials=n_trials)
print(study.best_params)
print(study.best_value)
print(study.best_trial)
else: # hyperparams optim
translate_closure(beam_width, nb_steps_ratio)
elif mode == "translate_attn":
log.info("writing translation + attention as html to %s" % dest_fn)
with cuda.get_device_from_id(gpu):
assert len(encdec_list) == 1
translations, attn_all = greedy_batch_translate(
encdec_list[0],
eos_idx,
src_data,
batch_size=mb_size,
gpu=gpu,
get_attention=True,
nb_steps=nb_steps,
use_chainerx=config_eval.process.use_chainerx)
# tgt_voc_with_unk = tgt_voc + ["#T_UNK#"]
# src_voc_with_unk = src_voc + ["#S_UNK#"]
assert len(translations) == len(src_data)
assert len(attn_all) == len(src_data)
attn_vis = AttentionVisualizer()
for num_t in six.moves.range(len(src_data)):
src_idx_list = src_data[num_t]
tgt_idx_list = translations[num_t][:-1]
attn = attn_all[num_t]
# assert len(attn) == len(tgt_idx_list)
src_w = src_indexer.deconvert_swallow(
src_idx_list, unk_tag="#S_UNK#") + ["SUM_ATTN"]
tgt_w = tgt_indexer.deconvert_swallow(tgt_idx_list,
unk_tag="#T_UNK#")
# src_w = [src_voc_with_unk[idx] for idx in src_idx_list] + ["SUM_ATTN"]
# tgt_w = [tgt_voc_with_unk[idx] for idx in tgt_idx_list]
# for j in six.moves.range(len(tgt_idx_list)):
# tgt_idx_list.append(tgt_voc_with_unk[t_and_attn[j][0]])
#
# print([src_voc_with_unk[idx] for idx in src_idx_list], tgt_idx_list)
attn_vis.add_plot(src_w, tgt_w, attn)
attn_vis.make_plot(dest_fn)
elif mode == "align":
import nmt_chainer.utilities.visualisation as visualisation
assert tgt_data is not None
assert len(tgt_data) == len(src_data)
log.info("writing alignment as html to %s" % dest_fn)
with cuda.get_device_from_id(gpu):
assert len(encdec_list) == 1
loss, attn_all = batch_align(
encdec_list[0],
eos_idx,
list(six.moves.zip(src_data, tgt_data)),
batch_size=mb_size,
gpu=gpu,
use_chainerx=config_eval.process.use_chainerx)
# tgt_voc_with_unk = tgt_voc + ["#T_UNK#"]
# src_voc_with_unk = src_voc + ["#S_UNK#"]
assert len(attn_all) == len(src_data)
plots_list = []
for num_t in six.moves.range(len(src_data)):
src_idx_list = src_data[num_t]
tgt_idx_list = tgt_data[num_t]
attn = attn_all[num_t]
# assert len(attn) == len(tgt_idx_list)
alignment = np.zeros((len(src_idx_list) + 1, len(tgt_idx_list)))
sum_al = [0] * len(tgt_idx_list)
for i in six.moves.range(len(src_idx_list)):
for j in six.moves.range(len(tgt_idx_list)):
alignment[i, j] = attn[j][i]
sum_al[j] += alignment[i, j]
for j in six.moves.range(len(tgt_idx_list)):
alignment[len(src_idx_list), j] = sum_al[j]
src_w = src_indexer.deconvert_swallow(
src_idx_list, unk_tag="#S_UNK#") + ["SUM_ATTN"]
tgt_w = tgt_indexer.deconvert_swallow(tgt_idx_list,
unk_tag="#T_UNK#")
# src_w = [src_voc_with_unk[idx] for idx in src_idx_list] + ["SUM_ATTN"]
# tgt_w = [tgt_voc_with_unk[idx] for idx in tgt_idx_list]
# for j in six.moves.range(len(tgt_idx_list)):
# tgt_idx_list.append(tgt_voc_with_unk[t_and_attn[j][0]])
#
# print([src_voc_with_unk[idx] for idx in src_idx_list], tgt_idx_list)
p1 = visualisation.make_alignment_figure(src_w, tgt_w, alignment)
# p2 = visualisation.make_alignment_figure(
# [src_voc_with_unk[idx] for idx in src_idx_list], tgt_idx_list, alignment)
plots_list.append(p1)
p_all = visualisation.Column(*plots_list)
visualisation.output_file(dest_fn)
visualisation.show(p_all)
# for t in translations_with_attn:
# for x, attn in t:
# print(x, attn)
# out.write(convert_idx_to_string([x for x, attn in t], tgt_voc + ["#T_UNK#"]) + "\n")
elif mode == "score_nbest":
log.info("opening nbest file %s" % nbest_to_rescore)
nbest_f = io.open(nbest_to_rescore, 'rt', encoding="utf8")
nbest_list = [[]]
for line in nbest_f:
line = line.strip().split("|||")
num_src = int(line[0].strip())
if num_src >= len(nbest_list):
assert num_src == len(nbest_list)
if max_nb_ex is not None and num_src >= max_nb_ex:
break
nbest_list.append([])
else:
assert num_src == len(nbest_list) - 1
sentence = line[1].strip()
nbest_list[-1].append(sentence.split(" "))
log.info("found nbest lists for %i source sentences" % len(nbest_list))
nbest_converted, make_data_infos = make_data.build_dataset_for_nbest_list_scoring(
tgt_indexer, nbest_list)
log.info("total %i sentences loaded" % make_data_infos.nb_ex)
log.info("#tokens src: %i of which %i (%f%%) are unknown" %
(make_data_infos.total_token, make_data_infos.total_count_unk,
float(make_data_infos.total_count_unk * 100) /
make_data_infos.total_token))
if len(nbest_list) != len(src_data[:max_nb_ex]):
log.warn("mismatch in lengths nbest vs src : %i != %i" %
(len(nbest_list), len(src_data[:max_nb_ex])))
assert len(nbest_list) == len(src_data[:max_nb_ex])
log.info("starting scoring")
from nmt_chainer.utilities import utils
res = []
for num in six.moves.range(len(nbest_converted)):
if num % 200 == 0:
print(num, file=sys.stderr)
elif num % 50 == 0:
print("*", file=sys.stderr)
res.append([])
src, tgt_list = src_data[num], nbest_converted[num]
src_batch, src_mask = utils.make_batch_src([src],
gpu=gpu,
volatile="on")
assert len(encdec_list) == 1
scorer = encdec_list[0].nbest_scorer(src_batch, src_mask)
nb_batches = (len(tgt_list) + mb_size - 1) // mb_size
for num_batch in six.moves.range(nb_batches):
tgt_batch, arg_sort = utils.make_batch_tgt(
tgt_list[num_batch * nb_batches:(num_batch + 1) *
nb_batches],
eos_idx=eos_idx,
gpu=gpu,
volatile="on",
need_arg_sort=True)
scores, attn = scorer(tgt_batch)
scores, _ = scores
scores = scores.data
assert len(arg_sort) == len(scores)
de_sorted_scores = [None] * len(scores)
for xpos in six.moves.range(len(arg_sort)):
original_pos = arg_sort[xpos]
de_sorted_scores[original_pos] = scores[xpos]
res[-1] += de_sorted_scores
print('', file=sys.stderr)
log.info("writing scores to %s" % dest_fn)
out = io.open(dest_fn, "wt", encoding="utf8")
for num in six.moves.range(len(res)):
for score in res[num]:
out.write("%i %f\n" % (num, score))
time_end = time.perf_counter()
translation_infos["loading_time"] = time_all_loaded - time_start
translation_infos["translation_time"] = time_end - time_all_loaded
translation_infos["total_time"] = time_end - time_start
if dest_fn is not None:
config_eval_session = config_eval.copy(readonly=False)
config_eval_session.add_section("translation_infos",
keep_at_bottom="metadata")
config_eval_session["translation_infos"] = translation_infos
config_eval_session.set_metadata_modified_time()
save_eval_config_fn = dest_fn + ".eval.config.json"
log.info("Saving eval config to %s" % save_eval_config_fn)
config_eval_session.save_to(save_eval_config_fn)
def beam_search_translate(
encdec,
eos_idx,
src_data,
beam_search_params: beam_search.BeamSearchParams = beam_search.
BeamSearchParams(),
#beam_width=20, beam_pruning_margin=None,
nb_steps=50,
gpu=None,
#beam_score_coverage_penalty=None, beam_score_coverage_penalty_strength=0.2,
need_attention=False,
nb_steps_ratio=None,
#beam_score_length_normalization='none', beam_score_length_normalization_strength=0.2,
post_score_length_normalization='simple',
post_score_length_normalization_strength=0.2,
post_score_coverage_penalty='none',
post_score_coverage_penalty_strength=0.2,
groundhog=False,
#force_finish=False,
prob_space_combination=False,
reverse_encdec=None,
#use_unfinished_translation_if_none_found=False,
nbest=None,
constraints_fn_list: Optional[List[
beam_search.BeamSearchConstraints]] = None,
use_astar=False,
astar_params: beam_search.AStarParams = beam_search.AStarParams(),
use_chainerx=False,
show_progress_bar=True):
nb_ex = len(src_data)
assert constraints_fn_list is None or len(constraints_fn_list) == nb_ex
if show_progress_bar:
range_creator = tqdm.trange
else:
range_creator = range
for num_ex in range_creator(nb_ex):
src_batch, src_mask = make_batch_src([src_data[num_ex]],
gpu=gpu,
use_chainerx=use_chainerx)
assert len(src_mask) == 0
if nb_steps_ratio is not None:
nb_steps = int(len(src_data[num_ex]) * nb_steps_ratio) + 1
# if isinstance(encdec, (tuple, list)):
# assert len(encdec) == 1
# encdec = encdec[0]
#
# translations = encdec.beam_search(src_batch, src_mask, nb_steps = nb_steps, eos_idx = eos_idx,
# beam_width = beam_width,
# beam_opt = beam_opt, need_attention = need_attention,
# groundhog = groundhog)
if not isinstance(encdec, (tuple, list)):
encdec = [encdec]
if constraints_fn_list is not None:
constraints_fn = constraints_fn_list[num_ex].get(
"ph_constraint", None)
else:
constraints_fn = None
translations = beam_search.ensemble_beam_search(
encdec,
src_batch,
src_mask,
nb_steps=nb_steps,
eos_idx=eos_idx,
beam_search_params=beam_search_params,
#beam_width=beam_width,
#beam_pruning_margin=beam_pruning_margin,
#beam_score_length_normalization=beam_score_length_normalization,
#beam_score_length_normalization_strength=beam_score_length_normalization_strength,
#beam_score_coverage_penalty=beam_score_coverage_penalty,
#beam_score_coverage_penalty_strength=beam_score_coverage_penalty_strength,
need_attention=need_attention,
#force_finish=force_finish,
prob_space_combination=prob_space_combination,
#use_unfinished_translation_if_none_found=use_unfinished_translation_if_none_found,
constraints=constraints_fn,
use_astar=use_astar,
astar_params=astar_params,
gpu=gpu)
# TODO: This is a quick patch, but actually ensemble_beam_search probably should not return empty translations except when no translation found
if len(translations) > 1:
translations = [t for t in translations if len(t[0]) > 0]
# print("nb_trans", len(translations), [score for _, score in translations])
# translations.sort(key = itemgetter(1), reverse = True)
if reverse_encdec is not None and len(translations) > 1:
rescored_translations = []
reverse_scores = reverse_rescore(reverse_encdec,
src_batch,
src_mask,
eos_idx,
[t[0] for t in translations],
gpu,
use_chainerx=use_chainerx)
for num_t in six.moves.range(len(translations)):
tr, sc, attn = translations[num_t]
rescored_translations.append(
(tr, sc + reverse_scores[num_t], attn))
translations = rescored_translations
xp = encdec[0].xp
if post_score_length_normalization == 'none' and post_score_coverage_penalty == 'none':
ranking_criterion = operator.itemgetter(1)
else:
ONE_ON_DEVICE = beam_search.convert_array_if_needed(
np.array(1.0, dtype=np.float32), xp, gpu)
def ranking_criterion(x):
length_normalization = 1
if post_score_length_normalization == 'simple':
length_normalization = len(x[0]) + 1
elif post_score_length_normalization == 'google':
length_normalization = pow(
(len(x[0]) + 5),
post_score_length_normalization_strength) / pow(
6, post_score_length_normalization_strength)
dic_score = 0
dic_score_computer = (constraints_fn_list[num_ex].get(
"dic_constraint", None) if constraints_fn_list is not None
else None)
if dic_score_computer is not None:
dic_score = dic_score_computer(x[0])
coverage_penalty = 0
if post_score_coverage_penalty == 'google':
assert len(src_data[num_ex]) == x[2][0].shape[0]
# log.info("sum={0}".format(sum(x[2])))
# log.info("min={0}".format(xp.minimum(sum(x[2]), xp.array(1.0))))
# log.info("log={0}".format(xp.log(xp.minimum(sum(x[2]), xp.array(1.0)))))
log_of_min_of_sum_over_j = xp.log(
xp.minimum(sum(x[2]), ONE_ON_DEVICE))
coverage_penalty = post_score_coverage_penalty_strength * xp.sum(
log_of_min_of_sum_over_j)
# log.info("cp={0}".format(coverage_penalty))
# cp = 0
# for i in six.moves.range(len(src_data[num_ex])):
# attn_sum = 0
# for j in six.moves.range(len(x[0])):
# attn_sum += x[2][j][i]
# #log.info("attn_sum={0}".format(attn_sum))
# #log.info("min={0}".format(min(attn_sum, 1.0)))
# #log.info("log={0}".format(math.log(min(attn_sum, 1.0))))
# cp += math.log(min(attn_sum, 1.0))
# log.info("cp={0}".format(cp))
# cp *= post_score_coverage_penalty_strength
# slow = x[1]/length_normalization + cp
# opti = x[1]/length_normalization + coverage_penalty
# log.info("type={0}....{1}".format(type(slow), type(opti)))
# log.info("shape={0} size={1} dim={2} data={3} elem={4}".format(opti.shape, opti.size, opti.ndim, opti.data, opti.item(0)))
# test = '!!!'
# if "{0}".format(slow) == "{0}".format(opti):
# test = ''
# log.info("score slow <=> optimized: {0} <=> {1} {2}".format(slow, opti, test))
return x[
1] / length_normalization + coverage_penalty + dic_score
translations.sort(key=ranking_criterion, reverse=True)
if nbest is not None:
yield translations[:nbest]
else:
yield [translations[0]]
def greedy_batch_translate(encdec,
eos_idx,
src_data,
batch_size=80,
gpu=None,
get_attention=False,
nb_steps=50,
reverse_src=False,
reverse_tgt=False,
use_chainerx=False):
with chainer.using_config("train", False), chainer.no_backprop_mode():
if encdec.encdec_type() == "ff":
result = encdec.greedy_batch_translate(src_data,
mb_size=batch_size,
nb_steps=nb_steps)
if get_attention:
dummy_attention = []
for src, tgt in six.moves.zip(src_data, result):
dummy_attention.append(
np.zeros((len(src), len(tgt)), dtype=np.float32))
return result, dummy_attention
else:
return result
nb_ex = len(src_data)
nb_batch = nb_ex // batch_size + (1 if nb_ex % batch_size != 0 else 0)
res = []
attn_all = []
for i in six.moves.range(nb_batch):
current_batch_raw_data = src_data[i * batch_size:(i + 1) *
batch_size]
if reverse_src:
current_batch_raw_data_new = []
for src_side in current_batch_raw_data:
current_batch_raw_data_new.append(src_side[::-1])
current_batch_raw_data = current_batch_raw_data_new
src_batch, src_mask = make_batch_src(current_batch_raw_data,
gpu=gpu,
use_chainerx=use_chainerx)
sample_greedy, score, attn_list = encdec(
src_batch,
nb_steps,
src_mask,
use_best_for_sample=True,
keep_attn_values=get_attention)
deb = de_batch(sample_greedy,
mask=None,
eos_idx=eos_idx,
is_variable=False)
res += deb
if get_attention:
deb_attn = de_batch(attn_list,
mask=None,
eos_idx=None,
is_variable=True,
raw=True,
reverse=reverse_tgt)
attn_all += deb_attn
if reverse_tgt:
new_res = []
for t in res:
if t[-1] == eos_idx:
new_res.append(t[:-1][::-1] + [t[-1]])
else:
new_res.append(t[::-1])
res = new_res
if get_attention:
assert not reverse_tgt, "not implemented"
return res, attn_all
else:
return res
def beam_search_translate(encdec,
eos_idx,
src_data,
beam_width=20,
beam_pruning_margin=None,
nb_steps=50,
gpu=None,
beam_score_coverage_penalty=None,
beam_score_coverage_penalty_strength=0.2,
need_attention=False,
nb_steps_ratio=None,
beam_score_length_normalization='none',
beam_score_length_normalization_strength=0.2,
post_score_length_normalization='simple',
post_score_length_normalization_strength=0.2,
post_score_coverage_penalty='none',
post_score_coverage_penalty_strength=0.2,
groundhog=False,
force_finish=False,
prob_space_combination=False,
reverse_encdec=None,
use_unfinished_translation_if_none_found=False,
nbest=None):
nb_ex = len(src_data)
for num_ex in range(nb_ex):
src_batch, src_mask = make_batch_src([src_data[num_ex]],
gpu=gpu,
volatile="on")
assert len(src_mask) == 0
if nb_steps_ratio is not None:
nb_steps = int(len(src_data[num_ex]) * nb_steps_ratio) + 1
# if isinstance(encdec, (tuple, list)):
# assert len(encdec) == 1
# encdec = encdec[0]
#
# translations = encdec.beam_search(src_batch, src_mask, nb_steps = nb_steps, eos_idx = eos_idx,
# beam_width = beam_width,
# beam_opt = beam_opt, need_attention = need_attention,
# groundhog = groundhog)
if not isinstance(encdec, (tuple, list)):
encdec = [encdec]
translations = beam_search.ensemble_beam_search(
encdec,
src_batch,
src_mask,
nb_steps=nb_steps,
eos_idx=eos_idx,
beam_width=beam_width,
beam_pruning_margin=beam_pruning_margin,
beam_score_length_normalization=beam_score_length_normalization,
beam_score_length_normalization_strength=
beam_score_length_normalization_strength,
beam_score_coverage_penalty=beam_score_coverage_penalty,
beam_score_coverage_penalty_strength=
beam_score_coverage_penalty_strength,
need_attention=need_attention,
force_finish=force_finish,
prob_space_combination=prob_space_combination,
use_unfinished_translation_if_none_found=
use_unfinished_translation_if_none_found)
# TODO: This is a quick patch, but actually ensemble_beam_search probably should not return empty translations except when no translation found
if len(translations) > 1:
translations = [t for t in translations if len(t[0]) > 0]
# print "nb_trans", len(translations), [score for _, score in translations]
# translations.sort(key = itemgetter(1), reverse = True)
if reverse_encdec is not None and len(translations) > 1:
rescored_translations = []
reverse_scores = reverse_rescore(reverse_encdec, src_batch,
src_mask, eos_idx,
[t[0] for t in translations], gpu)
for num_t in xrange(len(translations)):
tr, sc, attn = translations[num_t]
rescored_translations.append(
(tr, sc + reverse_scores[num_t], attn))
translations = rescored_translations
xp = encdec[0].xp
if post_score_length_normalization == 'none' and post_score_coverage_penalty == 'none':
ranking_criterion = operator.itemgetter(1)
else:
def ranking_criterion(x):
length_normalization = 1
if post_score_length_normalization == 'simple':
length_normalization = len(x[0]) + 1
elif post_score_length_normalization == 'google':
length_normalization = pow(
(len(x[0]) + 5),
post_score_length_normalization_strength) / pow(
6, post_score_length_normalization_strength)
coverage_penalty = 0
if post_score_coverage_penalty == 'google':
assert len(src_data[num_ex]) == x[2][0].shape[0]
# log.info("sum={0}".format(sum(x[2])))
# log.info("min={0}".format(xp.minimum(sum(x[2]), xp.array(1.0))))
# log.info("log={0}".format(xp.log(xp.minimum(sum(x[2]), xp.array(1.0)))))
log_of_min_of_sum_over_j = xp.log(
xp.minimum(sum(x[2]), xp.array(1.0)))
coverage_penalty = post_score_coverage_penalty_strength * xp.sum(
log_of_min_of_sum_over_j)
# log.info("cp={0}".format(coverage_penalty))
# cp = 0
# for i in xrange(len(src_data[num_ex])):
# attn_sum = 0
# for j in xrange(len(x[0])):
# attn_sum += x[2][j][i]
# #log.info("attn_sum={0}".format(attn_sum))
# #log.info("min={0}".format(min(attn_sum, 1.0)))
# #log.info("log={0}".format(math.log(min(attn_sum, 1.0))))
# cp += math.log(min(attn_sum, 1.0))
# log.info("cp={0}".format(cp))
# cp *= post_score_coverage_penalty_strength
# slow = x[1]/length_normalization + cp
# opti = x[1]/length_normalization + coverage_penalty
# log.info("type={0}....{1}".format(type(slow), type(opti)))
# log.info("shape={0} size={1} dim={2} data={3} elem={4}".format(opti.shape, opti.size, opti.ndim, opti.data, opti.item(0)))
# test = '!!!'
# if "{0}".format(slow) == "{0}".format(opti):
# test = ''
# log.info("score slow <=> optimized: {0} <=> {1} {2}".format(slow, opti, test))
return x[1] / length_normalization + coverage_penalty
translations.sort(key=ranking_criterion, reverse=True)
if nbest is not None:
yield translations[:nbest]
else:
yield [translations[0]]
def greedy_batch_translate(encdec,
eos_idx,
src_data,
batch_size=80,
gpu=None,
get_attention=False,
nb_steps=50,
reverse_src=False,
reverse_tgt=False):
nb_ex = len(src_data)
nb_batch = nb_ex / batch_size + (1 if nb_ex % batch_size != 0 else 0)
res = []
attn_all = []
for i in range(nb_batch):
current_batch_raw_data = src_data[i * batch_size:(i + 1) * batch_size]
if reverse_src:
current_batch_raw_data_new = []
for src_side in current_batch_raw_data:
current_batch_raw_data_new.append(src_side[::-1])
current_batch_raw_data = current_batch_raw_data_new
src_batch, src_mask = make_batch_src(current_batch_raw_data,
gpu=gpu,
volatile="on")
sample_greedy, score, attn_list = encdec(
src_batch,
nb_steps,
src_mask,
use_best_for_sample=True,
keep_attn_values=get_attention,
mode="test")
deb = de_batch(sample_greedy,
mask=None,
eos_idx=eos_idx,
is_variable=False)
res += deb
if get_attention:
deb_attn = de_batch(attn_list,
mask=None,
eos_idx=None,
is_variable=True,
raw=True,
reverse=reverse_tgt)
attn_all += deb_attn
if reverse_tgt:
new_res = []
for t in res:
if t[-1] == eos_idx:
new_res.append(t[:-1][::-1] + [t[-1]])
else:
new_res.append(t[::-1])
res = new_res
if get_attention:
assert not reverse_tgt, "not implemented"
return res, attn_all
else:
return res
def do_eval(config_eval):
src_fn = config_eval.process.src_fn
tgt_fn = config_eval.output.tgt_fn
mode = config_eval.method.mode
gpu = config_eval.process.gpu
dest_fn = config_eval.process.dest_fn
mb_size = config_eval.process.mb_size
nb_steps = config_eval.method.nb_steps
nb_steps_ratio = config_eval.method.nb_steps_ratio
max_nb_ex = config_eval.process.max_nb_ex
nbest_to_rescore = config_eval.output.nbest_to_rescore
nbest = config_eval.output.nbest
beam_width = config_eval.method.beam_width
beam_pruning_margin = config_eval.method.beam_pruning_margin
beam_score_length_normalization = config_eval.method.beam_score_length_normalization
beam_score_length_normalization_strength = config_eval.method.beam_score_length_normalization_strength
beam_score_coverage_penalty = config_eval.beam_score_coverage_penalty
beam_score_coverage_penalty_strength = config_eval.beam_score_coverage_penalty_strength
post_score_length_normalization = config_eval.method.post_score_length_normalization
post_score_length_normalization_strength = config_eval.method.post_score_length_normalization_strength
groundhog = config_eval.method.groundhog
tgt_unk_id = config_eval.output.tgt_unk_id
force_finish = config_eval.method.force_finish
prob_space_combination = config_eval.method.prob_space_combination
generate_attention_html = config_eval.output.generate_attention_html
rich_output_filename = config_eval.output.rich_output_filename
ref = config_eval.output.ref
dic = config_eval.output.dic
normalize_unicode_unk = config_eval.output.normalize_unicode_unk
attempt_to_relocate_unk_source = config_eval.output.attempt_to_relocate_unk_source
remove_unk = config_eval.output.remove_unk
post_score_coverage_penalty = config_eval.method.post_score_coverage_penalty
post_score_coverage_penalty_strength = config_eval.method.post_score_coverage_penalty_strength
time_start = time.clock()
encdec_list, eos_idx, src_indexer, tgt_indexer, reverse_encdec, model_infos_list = create_encdec(config_eval)
if config_eval.process.server is None:
eval_dir_placeholder = "@eval@/"
if dest_fn.startswith(eval_dir_placeholder):
if config_eval.trained_model is not None:
training_model_filename = config_eval.trained_model
else:
if len(config_eval.process.load_model_config) == 0:
log.error("Cannot detect value for $eval$ placeholder")
sys.exit(1)
training_model_filename = config_eval.process.load_model_config[0]
eval_dir = os.path.join(os.path.dirname(training_model_filename), "eval")
dest_fn = os.path.join(eval_dir, dest_fn[len(eval_dir_placeholder):])
log.info("$eval$ detected. dest_fn is: %s ", dest_fn)
ensure_path(eval_dir)
if src_fn is None:
(dev_src_from_config, dev_tgt_from_config, test_src_from_config, test_tgt_from_config) = get_src_tgt_dev_from_config_eval(config_eval)
if test_src_from_config is None:
log.error("Could not find value for source text, either on command line or in config files")
sys.exit(1)
log.info("using files from config as src:%s", test_src_from_config)
src_fn = test_src_from_config
if ref is None:
log.info("using files from config as ref:%s", test_tgt_from_config)
ref = test_tgt_from_config
log.info("opening source file %s" % src_fn)
src_data, stats_src_pp = build_dataset_one_side_pp(src_fn, src_pp=src_indexer,
max_nb_ex=max_nb_ex)
log.info("src data stats:\n%s", stats_src_pp.make_report())
if dest_fn is not None:
save_eval_config_fn = dest_fn + ".eval.init.config.json"
log.info("Saving initial eval config to %s" % save_eval_config_fn)
config_eval.save_to(save_eval_config_fn)
translation_infos = OrderedNamespace()
# log.info("%i sentences loaded" % make_data_infos.nb_ex)
# log.info("#tokens src: %i of which %i (%f%%) are unknown"%(make_data_infos.total_token,
# make_data_infos.total_count_unk,
# float(make_data_infos.total_count_unk * 100) /
# make_data_infos.total_token))
tgt_data = None
if tgt_fn is not None:
log.info("opening target file %s" % tgt_fn)
tgt_data, stats_tgt_pp = build_dataset_one_side_pp(tgt_fn, src_pp=tgt_indexer,
max_nb_ex=max_nb_ex)
log.info("tgt data stats:\n%s", stats_tgt_pp.make_report())
# log.info("%i sentences loaded"%make_data_infos.nb_ex)
# log.info("#tokens src: %i of which %i (%f%%) are unknown"%(make_data_infos.total_token,
# make_data_infos.total_count_unk,
# float(make_data_infos.total_count_unk * 100) /
# make_data_infos.total_token))
# translations = greedy_batch_translate(encdec, eos_idx, src_data, batch_size = mb_size, gpu = args.gpu)
translation_infos["src"] = src_fn
translation_infos["tgt"] = tgt_fn
translation_infos["ref"] = ref
for num_model, model_infos in enumerate(model_infos_list):
translation_infos["model%i" % num_model] = model_infos
time_all_loaded = time.clock()
if mode == "translate":
log.info("writing translation of to %s" % dest_fn)
with cuda.get_device(gpu):
assert len(encdec_list) == 1
translations = greedy_batch_translate(
encdec_list[0], eos_idx, src_data, batch_size=mb_size, gpu=gpu, nb_steps=nb_steps)
out = codecs.open(dest_fn, "w", encoding="utf8")
for t in translations:
if t[-1] == eos_idx:
t = t[:-1]
ct = tgt_indexer.deconvert(t, unk_tag="#T_UNK#")
# ct = convert_idx_to_string(t, tgt_voc + ["#T_UNK#"])
out.write(ct + "\n")
elif mode == "beam_search" or mode == "eval_bleu":
if config_eval.process.server is not None:
from nmt_chainer.translation.server import do_start_server
do_start_server(config_eval)
else:
translate_to_file_with_beam_search(dest_fn, gpu, encdec_list, eos_idx, src_data,
beam_width=beam_width,
beam_pruning_margin=beam_pruning_margin,
beam_score_coverage_penalty=beam_score_coverage_penalty,
beam_score_coverage_penalty_strength=beam_score_coverage_penalty_strength,
nb_steps=nb_steps,
nb_steps_ratio=nb_steps_ratio,
beam_score_length_normalization=beam_score_length_normalization,
beam_score_length_normalization_strength=beam_score_length_normalization_strength,
post_score_length_normalization=post_score_length_normalization,
post_score_length_normalization_strength=post_score_length_normalization_strength,
post_score_coverage_penalty=post_score_coverage_penalty,
post_score_coverage_penalty_strength=post_score_coverage_penalty_strength,
groundhog=groundhog,
tgt_unk_id=tgt_unk_id,
tgt_indexer=tgt_indexer,
force_finish=force_finish,
prob_space_combination=prob_space_combination,
reverse_encdec=reverse_encdec,
generate_attention_html=generate_attention_html,
src_indexer=src_indexer,
rich_output_filename=rich_output_filename,
use_unfinished_translation_if_none_found=True,
unprocessed_output_filename=dest_fn + ".unprocessed",
nbest=nbest)
translation_infos["dest"] = dest_fn
translation_infos["unprocessed"] = dest_fn + ".unprocessed"
if mode == "eval_bleu":
if ref is not None:
bc = bleu_computer.get_bc_from_files(ref, dest_fn)
print "bleu before unk replace:", bc
translation_infos["bleu"] = bc.bleu()
translation_infos["bleu_infos"] = str(bc)
else:
print "bleu before unk replace: No Ref Provided"
from nmt_chainer.utilities import replace_tgt_unk
replace_tgt_unk.replace_unk(dest_fn, src_fn, dest_fn + ".unk_replaced", dic, remove_unk,
normalize_unicode_unk,
attempt_to_relocate_unk_source)
translation_infos["unk_replaced"] = dest_fn + ".unk_replaced"
if ref is not None:
bc = bleu_computer.get_bc_from_files(ref, dest_fn + ".unk_replaced")
print "bleu after unk replace:", bc
translation_infos["post_unk_bleu"] = bc.bleu()
translation_infos["post_unk_bleu_infos"] = str(bc)
else:
print "bleu before unk replace: No Ref Provided"
elif mode == "translate_attn":
log.info("writing translation + attention as html to %s" % dest_fn)
with cuda.get_device(gpu):
assert len(encdec_list) == 1
translations, attn_all = greedy_batch_translate(
encdec_list[0], eos_idx, src_data, batch_size=mb_size, gpu=gpu,
get_attention=True, nb_steps=nb_steps)
# tgt_voc_with_unk = tgt_voc + ["#T_UNK#"]
# src_voc_with_unk = src_voc + ["#S_UNK#"]
assert len(translations) == len(src_data)
assert len(attn_all) == len(src_data)
attn_vis = AttentionVisualizer()
for num_t in xrange(len(src_data)):
src_idx_list = src_data[num_t]
tgt_idx_list = translations[num_t][:-1]
attn = attn_all[num_t]
# assert len(attn) == len(tgt_idx_list)
src_w = src_indexer.deconvert_swallow(src_idx_list, unk_tag="#S_UNK#") + ["SUM_ATTN"]
tgt_w = tgt_indexer.deconvert_swallow(tgt_idx_list, unk_tag="#T_UNK#")
# src_w = [src_voc_with_unk[idx] for idx in src_idx_list] + ["SUM_ATTN"]
# tgt_w = [tgt_voc_with_unk[idx] for idx in tgt_idx_list]
# for j in xrange(len(tgt_idx_list)):
# tgt_idx_list.append(tgt_voc_with_unk[t_and_attn[j][0]])
#
# print [src_voc_with_unk[idx] for idx in src_idx_list], tgt_idx_list
attn_vis.add_plot(src_w, tgt_w, attn)
attn_vis.make_plot(dest_fn)
elif mode == "align":
import nmt_chainer.utilities.visualisation as visualisation
assert tgt_data is not None
assert len(tgt_data) == len(src_data)
log.info("writing alignment as html to %s" % dest_fn)
with cuda.get_device(gpu):
assert len(encdec_list) == 1
loss, attn_all = batch_align(
encdec_list[0], eos_idx, zip(src_data, tgt_data), batch_size=mb_size, gpu=gpu)
# tgt_voc_with_unk = tgt_voc + ["#T_UNK#"]
# src_voc_with_unk = src_voc + ["#S_UNK#"]
assert len(attn_all) == len(src_data)
plots_list = []
for num_t in xrange(len(src_data)):
src_idx_list = src_data[num_t]
tgt_idx_list = tgt_data[num_t]
attn = attn_all[num_t]
# assert len(attn) == len(tgt_idx_list)
alignment = np.zeros((len(src_idx_list) + 1, len(tgt_idx_list)))
sum_al = [0] * len(tgt_idx_list)
for i in xrange(len(src_idx_list)):
for j in xrange(len(tgt_idx_list)):
alignment[i, j] = attn[j][i]
sum_al[j] += alignment[i, j]
for j in xrange(len(tgt_idx_list)):
alignment[len(src_idx_list), j] = sum_al[j]
src_w = src_indexer.deconvert_swallow(src_idx_list, unk_tag="#S_UNK#") + ["SUM_ATTN"]
tgt_w = tgt_indexer.deconvert_swallow(tgt_idx_list, unk_tag="#T_UNK#")
# src_w = [src_voc_with_unk[idx] for idx in src_idx_list] + ["SUM_ATTN"]
# tgt_w = [tgt_voc_with_unk[idx] for idx in tgt_idx_list]
# for j in xrange(len(tgt_idx_list)):
# tgt_idx_list.append(tgt_voc_with_unk[t_and_attn[j][0]])
#
# print [src_voc_with_unk[idx] for idx in src_idx_list], tgt_idx_list
p1 = visualisation.make_alignment_figure(
src_w, tgt_w, alignment)
# p2 = visualisation.make_alignment_figure(
# [src_voc_with_unk[idx] for idx in src_idx_list], tgt_idx_list, alignment)
plots_list.append(p1)
p_all = visualisation.Column(*plots_list)
visualisation.output_file(dest_fn)
visualisation.show(p_all)
# for t in translations_with_attn:
# for x, attn in t:
# print x, attn
# out.write(convert_idx_to_string([x for x, attn in t], tgt_voc + ["#T_UNK#"]) + "\n")
elif mode == "score_nbest":
log.info("opening nbest file %s" % nbest_to_rescore)
nbest_f = codecs.open(nbest_to_rescore, encoding="utf8")
nbest_list = [[]]
for line in nbest_f:
line = line.strip().split("|||")
num_src = int(line[0].strip())
if num_src >= len(nbest_list):
assert num_src == len(nbest_list)
if max_nb_ex is not None and num_src >= max_nb_ex:
break
nbest_list.append([])
else:
assert num_src == len(nbest_list) - 1
sentence = line[1].strip()
nbest_list[-1].append(sentence.split(" "))
log.info("found nbest lists for %i source sentences" % len(nbest_list))
nbest_converted, make_data_infos = make_data.build_dataset_for_nbest_list_scoring(tgt_indexer, nbest_list)
log.info("total %i sentences loaded" % make_data_infos.nb_ex)
log.info("#tokens src: %i of which %i (%f%%) are unknown" % (make_data_infos.total_token,
make_data_infos.total_count_unk,
float(make_data_infos.total_count_unk * 100) /
make_data_infos.total_token))
if len(nbest_list) != len(src_data[:max_nb_ex]):
log.warn("mismatch in lengths nbest vs src : %i != %i" % (len(nbest_list), len(src_data[:max_nb_ex])))
assert len(nbest_list) == len(src_data[:max_nb_ex])
log.info("starting scoring")
from nmt_chainer.utilities import utils
res = []
for num in xrange(len(nbest_converted)):
if num % 200 == 0:
print >>sys.stderr, num,
elif num % 50 == 0:
print >>sys.stderr, "*",
res.append([])
src, tgt_list = src_data[num], nbest_converted[num]
src_batch, src_mask = utils.make_batch_src([src], gpu=gpu, volatile="on")
assert len(encdec_list) == 1
scorer = encdec_list[0].nbest_scorer(src_batch, src_mask)
nb_batches = (len(tgt_list) + mb_size - 1) / mb_size
for num_batch in xrange(nb_batches):
tgt_batch, arg_sort = utils.make_batch_tgt(tgt_list[num_batch * nb_batches: (num_batch + 1) * nb_batches],
eos_idx=eos_idx, gpu=gpu, volatile="on", need_arg_sort=True)
scores, attn = scorer(tgt_batch)
scores, _ = scores
scores = scores.data
assert len(arg_sort) == len(scores)
de_sorted_scores = [None] * len(scores)
for xpos in xrange(len(arg_sort)):
original_pos = arg_sort[xpos]
de_sorted_scores[original_pos] = scores[xpos]
res[-1] += de_sorted_scores
print >>sys.stderr
log.info("writing scores to %s" % dest_fn)
out = codecs.open(dest_fn, "w", encoding="utf8")
for num in xrange(len(res)):
for score in res[num]:
out.write("%i %f\n" % (num, score))
time_end = time.clock()
translation_infos["loading_time"] = time_all_loaded - time_start
translation_infos["translation_time"] = time_end - time_all_loaded
translation_infos["total_time"] = time_end - time_start
if dest_fn is not None:
config_eval_session = config_eval.copy(readonly=False)
config_eval_session.add_section("translation_infos", keep_at_bottom="metadata")
config_eval_session["translation_infos"] = translation_infos
config_eval_session.set_metadata_modified_time()
save_eval_config_fn = dest_fn + ".eval.config.json"
log.info("Saving eval config to %s" % save_eval_config_fn)
config_eval_session.save_to(save_eval_config_fn)
