Python make_fts5_tokenizer Exemples

Exemple #1

Fichier : test_fts5.py Projet : kyodocn/sqlite-fts-python

def test_createtale_using_tokenizer_class(c):
    initialized = {}
    deleted = Counter()

    class ST(SimpleTokenizer):
        def __init__(self, context=None, args=None):
            initialized[self] = (context, tuple(args))

        def on_delete(self):
            deleted[self] += 1

    name = 'super_simple'
    fts5.register_tokenizer(c,
                            name,
                            fts5.make_fts5_tokenizer(ST),
                            context='test')
    sql = ("CREATE VIRTUAL TABLE fts "
           "USING FTS5(content, tokenize='{} {} {}')").format(
               name, 'arg', '引数')
    c.execute(sql)
    assert len(initialized) == 1
    assert list(initialized.values()) == [('test', ('arg', '引数'))]
    assert len(deleted) == 0
    sql = ("CREATE VIRTUAL TABLE fts_2 "
           "USING FTS5(content, tokenize='{} {} {}')").format(
               name, 'arg2', '引数2')
    c.execute(sql)
    c.close()
    assert set(initialized.values()) == {('test', ('arg', '引数')),
                                         ('test', ('arg2', '引数2'))}
    assert list(x for x in deleted.values()) == [1, 1]

Exemple #2

Fichier : test_fts5.py Projet : hideaki-t/sqlite-fts-python

def test_createtale_using_tokenizer_class(c):
    initialized = {}
    deleted = Counter()

    class ST(SimpleTokenizer):
        def __init__(self, context=None, args=None):
            initialized[self] = (context, tuple(args))

        def on_delete(self):
            deleted[self] += 1

    name = 'super_simple'
    fts5.register_tokenizer(
        c, name, fts5.make_fts5_tokenizer(ST), context='test')
    sql = (
        "CREATE VIRTUAL TABLE fts "
        "USING FTS5(content, tokenize='{} {} {}')").format(name, 'arg', '引数')
    c.execute(sql)
    assert len(initialized) == 1
    assert list(initialized.values()) == [('test', ('arg', '引数'))]
    assert len(deleted) == 0
    sql = (
        "CREATE VIRTUAL TABLE fts_2 "
        "USING FTS5(content, tokenize='{} {} {}')").format(name, 'arg2', '引数2')
    c.execute(sql)
    c.close()
    assert set(initialized.values()) == {('test', ('arg', '引数')),
                                         ('test', ('arg2', '引数2'))}
    assert list(x for x in deleted.values()) == [1, 1]

Exemple #3

Fichier : test_fts5.py Projet : kyodocn/sqlite-fts-python

def test_flags(c):
    flags_counter = Counter()

    class ST(SimpleTokenizer):
        def tokenize(self, text, flags):
            flags_counter[flags] += 1
            return super(ST, self).tokenize(text, flags)

    name = 'super_simple2'
    fts5.register_tokenizer(c, name, fts5.make_fts5_tokenizer(ST()))
    sql = ("CREATE VIRTUAL TABLE fts "
           "USING FTS5(content, tokenize='{}')").format(name)
    c.execute(sql)
    c.executemany('INSERT INTO fts VALUES(?)', [('abc def', ), ('abc xyz', ),
                                                ('あいうえお かきくけこ', ),
                                                ('あいうえお らりるれろ', )])
    c.execute("SELECT * FROM fts WHERE fts MATCH 'abc'").fetchall()
    c.execute("SELECT * FROM fts WHERE fts MATCH 'abc'").fetchall()
    c.close()
    assert flags_counter[fts5.FTS5_TOKENIZE_DOCUMENT] == 4
    assert flags_counter[fts5.FTS5_TOKENIZE_QUERY] == 2

Exemple #4

Fichier : test_fts5.py Projet : hideaki-t/sqlite-fts-python

def test_flags(c):
    flags_counter = Counter()

    class ST(SimpleTokenizer):
        def tokenize(self, text, flags):
            flags_counter[flags] += 1
            return super(ST, self).tokenize(text, flags)

    name = 'super_simple2'
    fts5.register_tokenizer(c, name, fts5.make_fts5_tokenizer(ST()))
    sql = ("CREATE VIRTUAL TABLE fts "
           "USING FTS5(content, tokenize='{}')").format(name)
    c.execute(sql)
    c.executemany('INSERT INTO fts VALUES(?)',
                  [('abc def', ), ('abc xyz', ), ('あいうえお かきくけこ', ),
                   ('あいうえお らりるれろ', )])
    c.execute("SELECT * FROM fts WHERE fts MATCH 'abc'").fetchall()
    c.execute("SELECT * FROM fts WHERE fts MATCH 'abc'").fetchall()
    c.close()
    assert flags_counter[fts5.FTS5_TOKENIZE_DOCUMENT] == 4
    assert flags_counter[fts5.FTS5_TOKENIZE_QUERY] == 2

Exemple #5

def create_table(c):
    fts.register_tokenizer(c, 'igo', fts.make_tokenizer_module(IgoTokenizer()))
    fts5.register_tokenizer(c, 'igo',
                            fts5.make_fts5_tokenizer(IgoTokenizer5()))
    c.execute("CREATE VIRTUAL TABLE fts USING FTS4(tokenize=igo)")
    c.execute("CREATE VIRTUAL TABLE fts5 USING FTS5(w, tokenize=igo)")

Exemple #6

def tm():
    return make_fts5_tokenizer(FTS3TokenizerAdaptor(SimpleFTS3Tokenizer()))

Exemple #7

Fichier : test_fts5.py Projet : kyodocn/sqlite-fts-python

def test_make_tokenizer_by_class(c):
    tm = fts5.make_fts5_tokenizer(SimpleTokenizer)
    assert all(
        getattr(tm, x) is not None
        for x in ('xCreate', 'xDelete', 'xTokenize'))
    c.close()

Exemple #8

Fichier : test_fts5.py Projet : kyodocn/sqlite-fts-python

def tm():
    return fts5.make_fts5_tokenizer(SimpleTokenizer())

Exemple #9

Fichier : test_fts5.py Projet : hideaki-t/sqlite-fts-python

def test_make_tokenizer_by_class(c):
    tm = fts5.make_fts5_tokenizer(SimpleTokenizer)
    assert all(
        getattr(tm, x) is not None
        for x in ('xCreate', 'xDelete', 'xTokenize'))
    c.close()

Exemple #10

Fichier : test_fts5.py Projet : hideaki-t/sqlite-fts-python

def tm():
    return fts5.make_fts5_tokenizer(SimpleTokenizer())

Exemple #11

def test_make_tokenizer(c):
    tm = fts5.make_fts5_tokenizer(SimpleTokenizer())
    assert all(
        getattr(tm, x) is not None
        for x in ("xCreate", "xDelete", "xTokenize"))
    c.close()