def test_dict_correct_preprocess():
"""did it create the correct dictionary also with preprocessing?"""
correct = {u"sentenc": 0, u"two": 2, u"one": 5, u"second": 3, u"of": 4, u"chap": 1, u"desc": 6}
create_corpus_run.main(create_corpus_pre_param_file)
d = Dictionary.load(path.join(out_dir, "dic1.dict"))
assert_equal(d.token2id, correct)
def test_dict_correct():
"""did it create the correct dictionary?"""
correct = {u"chap": 2, u"desc": 7, u"first": 6, u"of": 1, u"one": 3, u"second": 4, u"sentence": 0, u"two": 5}
create_corpus_run.main(create_corpus_param_file)
d = Dictionary.load(path.join(out_dir, "dic1.dict"))
assert_equal(d.token2id, correct)
def test_corpus_correct_preprocess():
"""is the corpus in the right format also with preprocessing?"""
global out_dir
create_corpus_run.main(create_corpus_pre_param_file)
correct = [[(0, 1.0), (6, 1.0)], [(0, 2.0), (1, 2.0), (2, 1.0), (3, 2.0), (4, 2.0), (5, 1.0), (6, 1.0)]]
cor = MmCorpus(path.join(out_dir, "corpus.mm"))
assert_equal(list(cor), correct)
def test_corpus_correct():
"""is the corpus in the right format?"""
global out_dir
create_corpus_run.main(create_corpus_param_file)
correct = [
[(0, 1.0), (6, 2.0), (7, 1.0)],
[(0, 2.0), (1, 2.0), (2, 2.0), (3, 1.0), (4, 2.0), (5, 1.0), (6, 1.0), (7, 1.0)],
]
cor = MmCorpus(path.join(out_dir, "corpus.mm"))
assert_equal(list(cor), correct)
def test_corpus_exists():
"""test whether all output files were created"""
global out_dir
create_corpus_run.main(create_corpus_param_file)
assert_true(path.exists(path.join(out_dir, "corpus.mm")))
assert_true(path.exists(path.join(out_dir, "dic1.dict")))