def test_constraints(train_params: str, beam_size: int, batch_size: int):
"""Task: copy short sequences of digits"""
with tmp_digits_dataset(prefix="test_constraints",
train_line_count=_TRAIN_LINE_COUNT,
train_max_length=_LINE_MAX_LENGTH,
dev_line_count=_DEV_LINE_COUNT,
dev_max_length=_LINE_MAX_LENGTH,
test_line_count=_TEST_LINE_COUNT,
test_line_count_empty=_TEST_LINE_COUNT_EMPTY,
test_max_length=_TEST_MAX_LENGTH,
sort_target=False) as data:
translate_params = " --batch-size {} --beam-size {}".format(
batch_size, beam_size)
# Ignore return values (perplexity and BLEU) for integration test
run_train_translate(train_params=train_params,
translate_params=translate_params,
translate_params_equiv=None,
train_source_path=data['source'],
train_target_path=data['target'],
dev_source_path=data['validation_source'],
dev_target_path=data['validation_target'],
test_source_path=data['test_source'],
test_target_path=data['test_target'],
max_seq_len=_LINE_MAX_LENGTH + C.SPACE_FOR_XOS,
work_dir=data['work_dir'],
use_prepared_data=False,
restrict_lexicon=False,
use_target_constraints=True)
def test_seq_copy(train_params: str,
translate_params: str,
restrict_lexicon: bool,
use_prepared_data: bool,
use_source_factors: bool,
use_constrained_decoding: bool):
"""Task: copy short sequences of digits"""
with tmp_digits_dataset(prefix="test_seq_copy",
train_line_count=_TRAIN_LINE_COUNT,
train_max_length=_LINE_MAX_LENGTH,
dev_line_count=_DEV_LINE_COUNT,
dev_max_length=_LINE_MAX_LENGTH,
test_line_count=_TEST_LINE_COUNT,
test_line_count_empty=_TEST_LINE_COUNT_EMPTY,
test_max_length=_TEST_MAX_LENGTH,
sort_target=False,
with_source_factors=use_source_factors,
with_target_constraints=use_constrained_decoding) as data:
# Only one of these is supported at a time in the tests
assert not (use_source_factors and use_constrained_decoding)
# When using source factors
train_source_factor_paths, dev_source_factor_paths, test_source_factor_paths = None, None, None
if use_source_factors:
train_source_factor_paths = [data['source']]
dev_source_factor_paths = [data['validation_source']]
test_source_factor_paths = [data['test_source']]
if use_constrained_decoding:
translate_params += " --json-input"
# Test model configuration, including the output equivalence of batch and no-batch decoding
translate_params_batch = translate_params + " --batch-size 2"
# Ignore return values (perplexity and BLEU) for integration test
run_train_translate(train_params=train_params,
translate_params=translate_params,
translate_params_equiv=translate_params_batch,
train_source_path=data['source'],
train_target_path=data['target'],
dev_source_path=data['validation_source'],
dev_target_path=data['validation_target'],
test_source_path=data['test_source'],
test_target_path=data['test_target'],
train_source_factor_paths=train_source_factor_paths,
dev_source_factor_paths=dev_source_factor_paths,
test_source_factor_paths=test_source_factor_paths,
max_seq_len=_LINE_MAX_LENGTH + C.SPACE_FOR_XOS,
restrict_lexicon=restrict_lexicon,
work_dir=data['work_dir'],
use_prepared_data=use_prepared_data)
def test_seq_copy(train_params: str, translate_params: str,
restrict_lexicon: bool, use_prepared_data: bool,
use_source_factors: bool):
"""Task: copy short sequences of digits"""
with tmp_digits_dataset(prefix="test_seq_copy",
train_line_count=_TRAIN_LINE_COUNT,
train_max_length=_LINE_MAX_LENGTH,
dev_line_count=_DEV_LINE_COUNT,
dev_max_length=_LINE_MAX_LENGTH,
test_line_count=_TEST_LINE_COUNT,
test_line_count_empty=_TEST_LINE_COUNT_EMPTY,
test_max_length=_TEST_MAX_LENGTH,
sort_target=False,
with_source_factors=use_source_factors) as data:
# When using source factors
train_source_factor_paths, dev_source_factor_paths, test_source_factor_paths = None, None, None
if use_source_factors:
train_source_factor_paths = [data['source']]
dev_source_factor_paths = [data['validation_source']]
test_source_factor_paths = [data['test_source']]
# Test model configuration, including the output equivalence of batch and no-batch decoding
if "--nbest-size" not in translate_params.split():
translate_params_batch = translate_params + " --batch-size 2"
else:
# nbest produces json output, which doesn't work with the splitting
# of translations and scores in run_train_translate, which in turn
# makes the comparison with the batch decoding fail.
# TODO: Refactor the run_train_translate function!
translate_params_batch = None
# Ignore return values (perplexity and BLEU) for integration test
run_train_translate(
train_params=train_params,
translate_params=translate_params,
translate_params_equiv=translate_params_batch,
train_source_path=data['source'],
train_target_path=data['target'],
dev_source_path=data['validation_source'],
dev_target_path=data['validation_target'],
test_source_path=data['test_source'],
test_target_path=data['test_target'],
train_source_factor_paths=train_source_factor_paths,
dev_source_factor_paths=dev_source_factor_paths,
test_source_factor_paths=test_source_factor_paths,
max_seq_len=_LINE_MAX_LENGTH + C.SPACE_FOR_XOS,
restrict_lexicon=restrict_lexicon,
work_dir=data['work_dir'],
use_prepared_data=use_prepared_data,
use_target_constraints=False)
def test_seq_copy(train_params: str, translate_params: str,
restrict_lexicon: bool, use_prepared_data: bool,
use_source_factors: bool, use_constrained_decoding: bool):
"""Task: copy short sequences of digits"""
with tmp_digits_dataset(
prefix="test_seq_copy",
train_line_count=_TRAIN_LINE_COUNT,
train_max_length=_LINE_MAX_LENGTH,
dev_line_count=_DEV_LINE_COUNT,
dev_max_length=_LINE_MAX_LENGTH,
test_line_count=_TEST_LINE_COUNT,
test_line_count_empty=_TEST_LINE_COUNT_EMPTY,
test_max_length=_TEST_MAX_LENGTH,
sort_target=False,
with_source_factors=use_source_factors,
with_target_constraints=use_constrained_decoding) as data:
# Only one of these is supported at a time in the tests
assert not (use_source_factors and use_constrained_decoding)
# When using source factors
train_source_factor_paths, dev_source_factor_paths, test_source_factor_paths = None, None, None
if use_source_factors:
train_source_factor_paths = [data['source']]
dev_source_factor_paths = [data['validation_source']]
test_source_factor_paths = [data['test_source']]
if use_constrained_decoding:
translate_params += " --json-input"
# Test model configuration, including the output equivalence of batch and no-batch decoding
translate_params_batch = translate_params + " --batch-size 2"
# Ignore return values (perplexity and BLEU) for integration test
run_train_translate(
train_params=train_params,
translate_params=translate_params,
translate_params_equiv=translate_params_batch,
train_source_path=data['source'],
train_target_path=data['target'],
dev_source_path=data['validation_source'],
dev_target_path=data['validation_target'],
test_source_path=data['test_target'],
test_target_path=data['test_target'],
train_source_factor_paths=train_source_factor_paths,
dev_source_factor_paths=dev_source_factor_paths,
test_source_factor_paths=test_source_factor_paths,
max_seq_len=_LINE_MAX_LENGTH + C.SPACE_FOR_XOS,
restrict_lexicon=restrict_lexicon,
work_dir=data['work_dir'],
use_prepared_data=use_prepared_data)
def test_other_clis(train_params: str, translate_params: str):
"""
Task: test CLIs and core features other than train & translate.
"""
with tmp_digits_dataset(prefix="test_other_clis",
train_line_count=_TRAIN_LINE_COUNT,
train_line_count_empty=_TRAIN_LINE_COUNT_EMPTY,
train_max_length=_LINE_MAX_LENGTH,
dev_line_count=_DEV_LINE_COUNT,
dev_max_length=_LINE_MAX_LENGTH,
test_line_count=_TEST_LINE_COUNT,
test_line_count_empty=_TEST_LINE_COUNT_EMPTY,
test_max_length=_TEST_MAX_LENGTH) as data:
# train a minimal default model
data = run_train_translate(train_params=train_params,
translate_params=translate_params,
data=data,
max_seq_len=_LINE_MAX_LENGTH +
C.SPACE_FOR_XOS)
_test_checkpoint_decoder(data['dev_source'], data['dev_target'],
data['model'])
_test_parameter_averaging(data['model'])
_test_extract_parameters_cli(data['model'])
_test_evaluate_cli(data['test_outputs'], data['test_target'])
def test_seq_copy(name, train_params, translate_params, use_prepared_data,
perplexity_thresh, bleu_thresh):
"""Task: copy short sequences of digits"""
with tmp_digits_dataset("test_seq_copy.",
_TRAIN_LINE_COUNT,
_LINE_MAX_LENGTH,
_DEV_LINE_COUNT,
_LINE_MAX_LENGTH,
_TEST_LINE_COUNT,
_TEST_LINE_COUNT_EMPTY,
_TEST_MAX_LENGTH,
seed_train=_SEED_TRAIN_DATA,
seed_dev=_SEED_DEV_DATA) as data:
# Test model configuration
perplexity, bleu, bleu_restrict, chrf = run_train_translate(
train_params,
translate_params,
None, # no second set of parameters
data['source'],
data['target'],
data['validation_source'],
data['validation_target'],
data['test_source'],
data['test_target'],
use_prepared_data=use_prepared_data,
max_seq_len=_LINE_MAX_LENGTH + 1,
restrict_lexicon=True,
work_dir=data['work_dir'],
seed=seed)
logger.info("test: %s", name)
logger.info("perplexity=%f, bleu=%f, bleu_restrict=%f chrf=%f",
perplexity, bleu, bleu_restrict, chrf)
assert perplexity <= perplexity_thresh
assert bleu >= bleu_thresh
assert bleu_restrict >= bleu_thresh
def test_seq_copy(train_params, translate_params, perplexity_thresh,
bleu_thresh):
"""Task: copy short sequences of digits"""
with TemporaryDirectory(prefix="test_seq_copy.") as work_dir:
# Simple digits files for train/dev data
train_source_path = os.path.join(work_dir, "train.src")
train_target_path = os.path.join(work_dir, "train.tgt")
dev_source_path = os.path.join(work_dir, "dev.src")
dev_target_path = os.path.join(work_dir, "dev.tgt")
generate_digits_file(train_source_path, train_target_path,
_TRAIN_LINE_COUNT, _LINE_MAX_LENGTH)
generate_digits_file(dev_source_path, dev_target_path, _DEV_LINE_COUNT,
_LINE_MAX_LENGTH)
# Test model configuration
perplexity, bleu = run_train_translate(train_params,
translate_params,
train_source_path,
train_target_path,
dev_source_path,
dev_target_path,
max_seq_len=_LINE_MAX_LENGTH +
1,
work_dir=work_dir)
assert perplexity <= perplexity_thresh
assert bleu >= bleu_thresh
def test_seq_sort(name, train_params, translate_params, use_prepared_data,
use_source_factor, perplexity_thresh, bleu_thresh):
"""Task: sort short sequences of digits"""
with tmp_digits_dataset("test_seq_sort.", _TRAIN_LINE_COUNT, _LINE_MAX_LENGTH, _DEV_LINE_COUNT, _LINE_MAX_LENGTH,
_TEST_LINE_COUNT, _TEST_LINE_COUNT_EMPTY, _TEST_MAX_LENGTH,
sort_target=True, seed_train=_SEED_TRAIN_DATA, seed_dev=_SEED_DEV_DATA,
with_source_factors=use_source_factor) as data:
# Test model configuration
perplexity, bleu, bleu_restrict, chrf = run_train_translate(train_params=train_params,
translate_params=translate_params,
translate_params_equiv=None,
train_source_path=data['source'],
train_target_path=data['target'],
dev_source_path=data['validation_source'],
dev_target_path=data['validation_target'],
test_source_path=data['test_source'],
test_target_path=data['test_target'],
train_source_factor_paths=data.get(
'train_source_factors'),
dev_source_factor_paths=data.get(
'dev_source_factors'),
test_source_factor_paths=data.get(
'test_source_factors'),
use_prepared_data=use_prepared_data,
max_seq_len=_LINE_MAX_LENGTH + C.SPACE_FOR_XOS,
restrict_lexicon=True,
work_dir=data['work_dir'],
seed=seed)
logger.info("test: %s", name)
logger.info("perplexity=%f, bleu=%f, bleu_restrict=%f chrf=%f", perplexity, bleu, bleu_restrict, chrf)
assert perplexity <= perplexity_thresh
assert bleu >= bleu_thresh
assert bleu_restrict >= bleu_thresh
def test_seq_sort(name, train_params, translate_params, use_prepared_data,
use_source_factor, perplexity_thresh, bleu_thresh):
"""Task: sort short sequences of digits"""
with tmp_digits_dataset("test_seq_sort.", _TRAIN_LINE_COUNT, _LINE_MAX_LENGTH, _DEV_LINE_COUNT, _LINE_MAX_LENGTH,
_TEST_LINE_COUNT, _TEST_LINE_COUNT_EMPTY, _TEST_MAX_LENGTH,
sort_target=True, seed_train=_SEED_TRAIN_DATA, seed_dev=_SEED_DEV_DATA,
with_source_factors=use_source_factor) as data:
# Test model configuration
perplexity, bleu, bleu_restrict, chrf = run_train_translate(train_params=train_params,
translate_params=translate_params,
translate_params_equiv=None,
train_source_path=data['source'],
train_target_path=data['target'],
dev_source_path=data['validation_source'],
dev_target_path=data['validation_target'],
test_source_path=data['test_source'],
test_target_path=data['test_target'],
train_source_factor_paths=data.get(
'train_source_factors'),
dev_source_factor_paths=data.get(
'dev_source_factors'),
test_source_factor_paths=data.get(
'test_source_factors'),
use_prepared_data=use_prepared_data,
max_seq_len=_LINE_MAX_LENGTH + C.SPACE_FOR_XOS,
restrict_lexicon=True,
work_dir=data['work_dir'],
seed=seed)
logger.info("test: %s", name)
logger.info("perplexity=%f, bleu=%f, bleu_restrict=%f chrf=%f", perplexity, bleu, bleu_restrict, chrf)
assert perplexity <= perplexity_thresh
assert bleu >= bleu_thresh
assert bleu_restrict >= bleu_thresh
def test_seq_sort(name, train_params, translate_params, perplexity_thresh,
bleu_thresh):
"""Task: sort short sequences of digits"""
with tmp_digits_dataset("test_seq_sort.",
_TRAIN_LINE_COUNT,
_LINE_MAX_LENGTH,
_DEV_LINE_COUNT,
_LINE_MAX_LENGTH,
sort_target=True,
seed_train=_SEED_TRAIN,
seed_dev=_SEED_DEV) as data:
# Test model configuration
perplexity, bleu, bleu_restrict = run_train_translate(
train_params,
translate_params,
None, # no second set of parameters
data['source'],
data['target'],
data['validation_source'],
data['validation_target'],
max_seq_len=_LINE_MAX_LENGTH + 1,
restrict_lexicon=True,
work_dir=data['work_dir'])
logger.info("test: %s", name)
logger.info("perplexity=%f, bleu=%f, bleu_restrict=%f", perplexity,
bleu, bleu_restrict)
assert perplexity <= perplexity_thresh
assert bleu >= bleu_thresh
assert bleu_restrict >= bleu_thresh
def test_seq_copy(train_params: str, translate_params: str, restrict_lexicon: bool):
"""Task: copy short sequences of digits"""
with tmp_digits_dataset("test_seq_copy", _TRAIN_LINE_COUNT, _LINE_MAX_LENGTH, _DEV_LINE_COUNT,
_LINE_MAX_LENGTH) as data:
# Test model configuration, including the output equivalence of batch and no-batch decoding
translate_params_batch = translate_params + " --batch-size 2"
# Ignore return values (perplexity and BLEU) for integration test
run_train_translate(train_params,
translate_params,
translate_params_batch,
data['source'],
data['target'],
data['validation_source'],
data['validation_target'],
max_seq_len=_LINE_MAX_LENGTH + 1,
restrict_lexicon=restrict_lexicon,
work_dir=data['work_dir'])
def test_seq_copy(train_params, translate_params):
"""Task: copy short sequences of digits"""
with TemporaryDirectory(prefix="test_seq_copy") as work_dir:
# Simple digits files for train/dev data
train_source_path = os.path.join(work_dir, "train.src")
train_target_path = os.path.join(work_dir, "train.tgt")
dev_source_path = os.path.join(work_dir, "dev.src")
dev_target_path = os.path.join(work_dir, "dev.tgt")
generate_digits_file(train_source_path, train_target_path, _TRAIN_LINE_COUNT, _LINE_MAX_LENGTH)
generate_digits_file(dev_source_path, dev_target_path, _DEV_LINE_COUNT, _LINE_MAX_LENGTH)
# Test model configuration
# Ignore return values (perplexity and BLEU) for integration test
run_train_translate(train_params,
translate_params,
train_source_path,
train_target_path,
dev_source_path,
dev_target_path,
max_seq_len=_LINE_MAX_LENGTH + 1,
work_dir=work_dir)
def test_constraints(train_params: str, translate_params: str):
with tmp_digits_dataset(prefix="test_constraints",
train_line_count=_TRAIN_LINE_COUNT,
train_max_length=_LINE_MAX_LENGTH,
dev_line_count=_DEV_LINE_COUNT,
dev_max_length=_LINE_MAX_LENGTH,
test_line_count=_TEST_LINE_COUNT,
test_line_count_empty=_TEST_LINE_COUNT_EMPTY,
test_max_length=_TEST_MAX_LENGTH,
sort_target=False) as data:
# train a minimal default model
data = run_train_translate(train_params=train_params, translate_params=translate_params, data=data,
max_seq_len=_LINE_MAX_LENGTH + C.SPACE_FOR_XOS)
# 'constraint' = positive constraints (must appear), 'avoid' = negative constraints (must not appear)
for constraint_type in ["constraints", "avoid"]:
_test_constrained_type(constraint_type=constraint_type, data=data, translate_params=translate_params)