def test_tuples_and_strings(): tuples = [("a", "b"), ("c", ), ("de", "fg")] strings = ["a b", "c", "de fg"] assert tokenization.tuple_sequence_to_strings(tuples) == strings assert tokenization.tuple_sequence_to_strings(strings) == strings assert tokenization.string_sequence_to_tuples(strings) == tuples assert tokenization.string_sequence_to_tuples(tuples) == tuples
def test_tokenizing_pipeline(voc_mapping, with_frequencies): tok = tokenization.tokenizing_pipeline_from_vocabulary_file( VOCABULARY_FILE, voc_mapping=voc_mapping ) if not with_frequencies: tok.frequencies = None if voc_mapping == {}: assert tok("the working memory group xyzzzz groups") == [ "working memory", "group", "groups", ] else: assert tok("the working memory group xyzzzz groups") == [ "working memory", "group", "group", ] assert tok.get_full_vocabulary( as_tuples=True ) == tokenization.string_sequence_to_tuples(tok.get_full_vocabulary()) assert tok.get_vocabulary( as_tuples=True ) == tokenization.string_sequence_to_tuples(tok.get_vocabulary()) if voc_mapping == "auto": assert len(tok.get_full_vocabulary()) == len(tok.get_vocabulary()) + 2 else: assert len(tok.get_full_vocabulary()) == len(tok.get_vocabulary()) assert len(tok.get_frequencies()) == len(tok.get_vocabulary()) if with_frequencies: assert hasattr(tok, "frequencies_") assert len(tok.get_frequencies()) == len(tok.get_vocabulary()) with tempfile.TemporaryDirectory() as tmp_dir: voc_file = os.path.join(tmp_dir, "voc_file.csv") tok.to_vocabulary_file(voc_file) loaded = tokenization.tokenizing_pipeline_from_vocabulary_file( voc_file, voc_mapping=voc_mapping ) assert ( loaded.vocabulary_mapping_.voc_mapping == tok.vocabulary_mapping_.voc_mapping ) assert loaded.get_full_vocabulary() == tok.get_full_vocabulary() assert loaded.get_vocabulary() == tok.get_vocabulary()