Пример #1
0
def test_corpus_fkgl():
    assert corpus_fkgl(
        get_orig_sents('turkcorpus_test_legacy')) == pytest.approx(9.9,
                                                                   abs=1e-1)
    assert corpus_fkgl(
        get_refs_sents('turkcorpus_test_legacy')[0]) == pytest.approx(8.2,
                                                                      abs=1e-1)
Пример #2
0
def test_corpus_sari():
    orig_sents = get_orig_sents('turkcorpus_test')
    refs_sents = get_refs_sents('turkcorpus_test')
    system_outputs_dir = get_system_outputs_dir('turkcorpus_test')
    hyp_sents = read_lines(system_outputs_dir / "ACCESS")
    sari_score = sari.corpus_sari(orig_sents, hyp_sents, refs_sents)
    assert sari_score == pytest.approx(
        41.381013)  # Scores from MUSS https://arxiv.org/abs/2005.00352
Пример #3
0
def test_corpus_samsa():
    orig_sents = get_orig_sents('qats_test')
    refs_sents = get_refs_sents('qats_test')
    samsa_score = corpus_samsa(orig_sents,
                               refs_sents[0],
                               lowercase=False,
                               tokenizer='moses')
    assert samsa_score == pytest.approx(36.94996509406232)
Пример #4
0
def test_corpus_sari_legacy():
    orig_sents = get_orig_sents('turkcorpus_test_legacy')
    refs_sents = get_refs_sents('turkcorpus_test_legacy')
    system_outputs_dir = get_system_outputs_dir('turkcorpus_test')

    hyp_sents = read_lines(system_outputs_dir / "tok.low/Dress-Ls.tok.low")
    sari_score = sari.corpus_sari(orig_sents,
                                  hyp_sents,
                                  refs_sents,
                                  legacy=True)
    assert sari_score == pytest.approx(37.266058818588216)

    hyp_sents = read_lines(system_outputs_dir / "tok.low/Dress.tok.low")
    sari_score = sari.corpus_sari(orig_sents,
                                  hyp_sents,
                                  refs_sents,
                                  legacy=True)
    assert sari_score == pytest.approx(37.08210095744638)

    hyp_sents = read_lines(system_outputs_dir / "tok.low/EncDecA.tok.low")
    sari_score = sari.corpus_sari(orig_sents,
                                  hyp_sents,
                                  refs_sents,
                                  legacy=True)
    assert sari_score == pytest.approx(35.65754396121206)

    hyp_sents = read_lines(system_outputs_dir / "tok.low/Hybrid.tok.low")
    sari_score = sari.corpus_sari(orig_sents,
                                  hyp_sents,
                                  refs_sents,
                                  legacy=True)
    assert sari_score == pytest.approx(31.39665078989411)

    hyp_sents = read_lines(system_outputs_dir / "tok.low/PBMT-R.tok.low")
    sari_score = sari.corpus_sari(orig_sents,
                                  hyp_sents,
                                  refs_sents,
                                  legacy=True)
    assert sari_score == pytest.approx(38.558843050332037)

    hyp_sents = read_lines(system_outputs_dir / "tok.low/SBMT-SARI.tok.low")
    sari_score = sari.corpus_sari(orig_sents,
                                  hyp_sents,
                                  refs_sents,
                                  legacy=True)
    assert sari_score == pytest.approx(39.964857928109127)
Пример #5
0
def get_orig_and_refs_sents(test_set,
                            orig_sents_path=None,
                            refs_sents_paths=None):
    # Get original and reference sentences
    if test_set == 'custom':
        assert orig_sents_path is not None
        assert refs_sents_paths is not None
        if type(refs_sents_paths) == str:
            refs_sents_paths = refs_sents_paths.split(',')
        orig_sents = read_lines(orig_sents_path)
        refs_sents = [
            read_lines(ref_sents_path) for ref_sents_path in refs_sents_paths
        ]
    else:
        orig_sents = get_orig_sents(test_set)
        refs_sents = get_refs_sents(test_set)
    # Final checks
    assert all([len(orig_sents) == len(ref_sents) for ref_sents in refs_sents])
    return orig_sents, refs_sents
Пример #6
0
def get_orig_and_refs_sents(test_set,
                            orig_sents_path=None,
                            refs_sents_paths=None):
    # Get original and reference sentences
    if test_set == "custom":
        assert orig_sents_path is not None
        assert refs_sents_paths is not None
        if type(refs_sents_paths) == str:
            refs_sents_paths = refs_sents_paths.split(",")
        orig_sents = read_lines(orig_sents_path)
        refs_sents = [
            read_lines(ref_sents_path) for ref_sents_path in refs_sents_paths
        ]
    else:
        orig_sents = get_orig_sents(test_set)
        refs_sents = get_refs_sents(test_set)
    # Final checks
    assert all(
        [len(orig_sents) == len(ref_sents) for ref_sents in refs_sents]
    ), f'Not same number of lines for test_set={test_set}, orig_sents_path={orig_sents_path}, refs_sents_paths={refs_sents_paths}'  # noqa: E501
    return orig_sents, refs_sents