def translate_and_compute_bleu(model, params, subtokenizer, bleu_source,
bleu_ref):
"""Translate file and report the cased and uncased bleu scores.
Args:
model: A Keras model, used to generate the translations.
params: A dictionary, containing the translation related parameters.
subtokenizer: A subtokenizer object, used for encoding and decoding source
and translated lines.
bleu_source: A file containing source sentences for translation.
bleu_ref: A file containing the reference for the translated sentences.
distribution_strategy: A platform distribution strategy, used for TPU based
translation.
Returns:
uncased_score: A float, the case insensitive BLEU score.
cased_score: A float, the case sensitive BLEU score.
"""
# Create temporary file to store translation.
output = 'out'
translate.translate_file(model,
params,
subtokenizer,
bleu_source,
output_file=output,
print_all_translations=False)
# Compute uncased and cased bleu scores.
uncased_score = compute_bleu.bleu_wrapper(bleu_ref, output, False)
cased_score = compute_bleu.bleu_wrapper(bleu_ref, output, True)
os.remove(output)
return uncased_score, cased_score
def translate_and_compute_bleu(model, subtokenizer, bleu_source, bleu_ref):
"""
Translate file and report the cased and uncased bleu scores
"""
tmp = tempfile.NamedTemporaryFile(delete=False)
tmp_filename = tmp.name
translate.translate_file(model, subtokenizer, bleu_source, output_file=tmp_filename,
print_all_translations=False)
uncased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, False)
os.remove(tmp_filename)
return uncased_score
def translate_and_compute_bleu(estimator, subtokenizer, bleu_source, bleu_ref):
"""Translate file and report the cased and uncased bleu scores."""
# Create temporary file to store translation.
tmp = tempfile.NamedTemporaryFile(delete=False)
tmp_filename = tmp.name
translate.translate_file(
estimator, subtokenizer, bleu_source, output_file=tmp_filename,
print_all_translations=False)
# Compute uncased and cased bleu scores.
uncased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, False)
cased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, True)
os.remove(tmp_filename)
return uncased_score, cased_score
def translate_and_compute_bleu(estimator, subtokenizer, bleu_source, bleu_ref):
"""Translate file and report the cased and uncased bleu scores."""
# Create temporary file to store translation.
tmp = tempfile.NamedTemporaryFile(delete=False)
tmp_filename = tmp.name
translate.translate_file(
estimator, subtokenizer, bleu_source, output_file=tmp_filename,
print_all_translations=False)
# Compute uncased and cased bleu scores.
uncased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, False)
cased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, True)
os.remove(tmp_filename)
return uncased_score, cased_score
def test_bleu(model, fields, epoch):
translator = nmt.Translator(model, fields, opt.beam_size, 1,
opt.decode_max_length, None, use_cuda)
src_fin = opt.multi_bleu_src
tgt_fout = os.path.join(opt.out_dir, 'translate.epoch%d' % (epoch))
translate_file(translator, src_fin, tgt_fout, fields, use_cuda)
output = os.popen('perl %s/tools/multi-bleu.pl %s < %s' %
(args.nmt_dir, ' '.join(opt.multi_bleu_refs), tgt_fout))
output = output.read()
# Get bleu value
bleu_val = re.findall('BLEU = (.*?),', output, re.S)[0]
bleu_val = float(bleu_val)
return bleu_val
def compile_file(path, filename=None):
'''
compile tempy file into compiled python bytecode(.pyc file)
'''
if filename is None:
filename = path
stmts = translate_file(path, filename=filename)
return _compile_kont(stmts, filename)
def compile_file(path, filename=None):
'''
compile tempy file into compiled python bytecode(.pyc file)
'''
if filename is None:
filename = path
stmts = translate_file(path, filename=filename)
return _compile_kont(stmts, filename)
def translate_and_compute_bleu(estimator, subtokenizer, subtokenizer_target, bleu_source, bleu_ref):
"""Translate file and report the cased and uncased bleu scores."""
# Create temporary file to store translation.
tmp = tempfile.NamedTemporaryFile(delete=False)
tmp_filename = tmp.name
#如果不用close()，remove时会出现PermissionError另一个程序正在使用此文件，进程无法访问。
tmp.close()
translate.translate_file(
estimator, subtokenizer, subtokenizer_target, bleu_source, output_file=tmp_filename,
print_all_translations=False)
# Compute uncased and cased bleu scores.
uncased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, False)
cased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, True)
# tmp.close()
os.remove(tmp_filename)
return uncased_score, cased_score
def translate_and_compute_bleu(model,
params,
subtokenizer,
bleu_source,
bleu_ref,
distribution_strategy=None):
"""Translate file and report the cased and uncased bleu scores.
Args:
model: A Keras model, used to generate the translations.
params: A dictionary, containing the translation related parameters.
subtokenizer: A subtokenizer object, used for encoding and decoding source
and translated lines.
bleu_source: A file containing source sentences for translation.
bleu_ref: A file containing the reference for the translated sentences.
distribution_strategy: A platform distribution strategy, used for TPU based
translation.
Returns:
uncased_score: A float, the case insensitive BLEU score.
cased_score: A float, the case sensitive BLEU score.
"""
# Create temporary file to store translation.
tmp = tempfile.NamedTemporaryFile(delete=False)
tmp_filename = tmp.name
translate.translate_file(model,
params,
subtokenizer,
bleu_source,
output_file=tmp_filename,
print_all_translations=False,
distribution_strategy=distribution_strategy)
# Compute uncased and cased bleu scores.
uncased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, False)
cased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, True)
# os.remove(tmp_filename)
try:
os.remove(tmp_filename)
except OSError:
pass
return uncased_score, cased_score
def _code_generation(self, tpy_path, tpyc_path, write_to_pyc=True):
if self.compile_option.write_py:
py_path = _exchange_ext(tpyc_path, "py")
try:
with open(py_path, "w") as f:
f.write(pystmts_to_string(translate_file(tpy_path)))
except IOError as err:
self.compile_option.log("IOError occured while writing .py file(%s): %s"%(tpyc_path, str(err)))
code = compile_file(tpy_path)
if write_to_pyc:
try:
_write_code(tpyc_path, code)
except IOError as err:
self.compile_option.log("IOError occured while writing codeobject to .tpyc file(%s): %s"%(tpyc_path, str(err)))
return code
def _code_generation(self, tpy_path, tpyc_path, write_to_pyc=True):
if self.compile_option.write_py:
py_path = _exchange_ext(tpyc_path, "py")
try:
with open(py_path, "w") as f:
f.write(pystmts_to_string(translate_file(tpy_path)))
except IOError as err:
self.compile_option.log(
"IOError occured while writing .py file(%s): %s" %
(tpyc_path, str(err)))
code = compile_file(tpy_path)
if write_to_pyc:
try:
_write_code(tpyc_path, code)
except IOError as err:
self.compile_option.log(
"IOError occured while writing codeobject to .tpyc file(%s): %s"
% (tpyc_path, str(err)))
return code
#!/usr/bin/python3
# cbc.py
# Note: this is an interpreter, not a compiler - the name just sounds better this way.
import os
import sys
import translate as trt
import run_erase as rpf
if len(sys.argv) > 2 or len(sys.argv) == 1:
print("Specify a file.")
exit()
in_file = str(sys.argv[1])
if (in_file.endswith(".cboy")) is False:
print("It's a good habit to use the .cboy file extension.")
in_file = sys.argv[1]
out_file = "text_out.py"
trt.translate_file(in_file)
rpf.run_file(out_file)
rpf.erase_file(out_file)
