def validate_with_script(session, beam_search_sampler):
config = beam_search_sampler.configs[0]
if config.valid_script == None:
return None
logging.info('Starting external validation.')
out = tempfile.NamedTemporaryFile(mode='w')
translate_utils.translate_file(
input_file=open(config.valid_bleu_source_dataset, encoding="UTF-8"),
output_file=out,
session=session,
sampler=beam_search_sampler,
config=config,
max_translation_len=config.translation_maxlen,
normalization_alpha=config.normalization_alpha,
consts_config_str=None,
nbest=False,
minibatch_size=config.valid_batch_size,
)
out.flush()
args = [config.valid_script, out.name]
proc = subprocess.Popen(args,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout_bytes, stderr_bytes = proc.communicate()
encoding = locale.getpreferredencoding()
stdout = stdout_bytes.decode(encoding=encoding)
stderr = stderr_bytes.decode(encoding=encoding)
if len(stderr) > 0:
logging.info("Validation script wrote the following to standard "
"error:\n" + stderr)
if proc.returncode != 0:
logging.warning("Validation script failed (returned exit status of "
"{}).".format(proc.returncode))
return None
try:
score = float(stdout.split()[0])
except:
logging.warning("Validation script output does not look like a score: "
"{}".format(stdout))
return None
logging.info("Validation script score: {}".format(score))
return score
def main(settings):
"""
Translates a source language file (or STDIN) into a target language file
(or STDOUT).
"""
# Create the TensorFlow session.
g = tf.Graph()
with g.as_default():
tf_config = tf.compat.v1.ConfigProto()
tf_config.allow_soft_placement = True
session = tf.compat.v1.Session(config=tf_config)
# Load config file for each model.
configs = []
for model in settings.models:
config = load_config_from_json_file(model)
setattr(config, 'reload', model)
setattr(config, 'translation_maxlen', settings.translation_maxlen)
configs.append(config)
# Create the model graphs.
logging.debug("Loading models\n")
models = []
for i, config in enumerate(configs):
with tf.compat.v1.variable_scope("model%d" % i) as scope:
if config.model_type == "transformer":
model = TransformerModel(
config, consts_config_str=settings.config_str)
else:
model = rnn_model.RNNModel(config)
model.sampling_utils = SamplingUtils(settings)
models.append(model)
# Add smoothing variables (if the models were trained with smoothing).
# FIXME Assumes either all models were trained with smoothing or none were.
if configs[0].exponential_smoothing > 0.0:
smoothing = ExponentialSmoothing(configs[0].exponential_smoothing)
# Restore the model variables.
for i, config in enumerate(configs):
with tf.compat.v1.variable_scope("model%d" % i) as scope:
_ = model_loader.init_or_restore_variables(
config, session, ensemble_scope=scope)
# Swap-in the smoothed versions of the variables.
if configs[0].exponential_smoothing > 0.0:
session.run(fetches=smoothing.swap_ops)
max_translation_len = settings.translation_maxlen
# Create a BeamSearchSampler / RandomSampler.
if settings.translation_strategy == 'beam_search':
sampler = BeamSearchSampler(models, configs, settings.beam_size)
else:
assert settings.translation_strategy == 'sampling'
sampler = RandomSampler(models, configs, settings.beam_size)
# Warn about the change from neg log probs to log probs for the RNN.
if settings.n_best:
model_types = [config.model_type for config in configs]
if 'rnn' in model_types:
logging.warn(
'n-best scores for RNN models have changed from '
'positive to negative (as of commit 95793196...). '
'If you are using the scores for reranking etc, then '
'you may need to update your scripts.')
# Translate the source file.
translate_utils.translate_file(
input_file=settings.input,
output_file=settings.output,
session=session,
sampler=sampler,
config=configs[0],
max_translation_len=max_translation_len,
normalization_alpha=settings.normalization_alpha,
consts_config_str=settings.config_str,
nbest=settings.n_best,
minibatch_size=settings.minibatch_size,
maxibatch_size=settings.maxibatch_size)