def _test_checkpoint_decoder(dev_source_path: str, dev_target_path: str,
model_path: str):
"""
Runs checkpoint decoder on 10% of the dev data and checks whether metric keys are present in the result dict.
"""
with open(dev_source_path) as dev_fd:
num_dev_sent = sum(1 for _ in dev_fd)
sample_size = min(1, int(num_dev_sent * 0.1))
model, source_vocabs, target_vocab = load_model(model_folder=model_path,
context=[mx.cpu()])
cp_decoder = sockeye.checkpoint_decoder.CheckpointDecoder(
context=mx.cpu(),
inputs=[dev_source_path],
references=dev_target_path,
source_vocabs=source_vocabs,
target_vocab=target_vocab,
model=model,
model_folder=model_path,
sample_size=sample_size,
batch_size=2,
beam_size=2)
cp_metrics = cp_decoder.decode_and_evaluate()
logger.info("Checkpoint decoder metrics: %s", cp_metrics)
assert 'bleu' in cp_metrics
assert 'chrf' in cp_metrics
assert 'decode-walltime' in cp_metrics
def _test_checkpoint_decoder(dev_source_path: str, dev_target_path: str,
model_path: str):
"""
Runs checkpoint decoder on 10% of the dev data and checks whether metric
keys are present in the result dict. Also checks that we can reload model
parameters after running the checkpoint decoder (case when using the
plateau-reduce scheduler).
"""
with open(dev_source_path) as dev_fd:
num_dev_sent = sum(1 for _ in dev_fd)
sample_size = min(1, int(num_dev_sent * 0.1))
model, source_vocabs, target_vocabs = load_model(model_folder=model_path,
device=pt.device('cpu'))
cp_decoder = sockeye.checkpoint_decoder.CheckpointDecoder(
device=pt.device('cpu'),
inputs=[dev_source_path],
references=[dev_target_path],
source_vocabs=source_vocabs,
target_vocabs=target_vocabs,
model=model,
model_folder=model_path,
sample_size=sample_size,
batch_size=2,
beam_size=2)
cp_metrics = cp_decoder.decode_and_evaluate()
logger.info("Checkpoint decoder metrics: %s", cp_metrics)
assert 'bleu' in cp_metrics
assert 'chrf' in cp_metrics
assert 'decode-walltime' in cp_metrics
model.load_parameters(os.path.join(model_path, C.PARAMS_BEST_NAME),
device=pt.device('cpu'))
def _test_mc_dropout(model_path: str):
"""
Check that loading a model with MC Dropoout returns a model with dropout layers.
"""
model, _, _ = load_model(model_folder=model_path,
context=[mx.cpu()],
mc_dropout=True,
inference_only=True,
hybridize=True)
# Ensure the model has some dropout turned on
config_blocks = [
block for _, block in model.config.__dict__.items()
if isinstance(block, Config)
]
dropout_settings = {
setting: val
for block in config_blocks for setting, val in block.__dict__.items()
if "dropout" in setting
}
assert any(s > 0.0 for s in dropout_settings.values())