Python create_shlex_tokenizer示例

示例#1

 def get_default_tokenizer(cls):
     return tokenizers.create_shlex_tokenizer(
         with_bof=True,
         with_eof=True,
         wordchars=string.ascii_uppercase + string.ascii_lowercase +
         string.digits + '_.',
     )

示例#2

def test_basic_tokenizer_yields_token_and_lookahead():
    tokenizer = tokenizers.create_shlex_tokenizer()

    tokens = list(ShiftReduceParser.tokenize_with_lookahead(tokenizer, ''))
    assert len(tokens) == 0

    tokens = list(ShiftReduceParser.tokenize_with_lookahead(
        tokenizer, 'a b c'))
    assert len(tokens) == 3
    assert tokens[0] == ('a', 'b')
    assert tokens[1] == ('b', 'c')
    assert tokens[2] == ('c', None)

示例#3

def test_tokenizer_with_bof_yields_token_and_lookahead():
    tokenizer = tokenizers.create_shlex_tokenizer(with_bof=True)

    tokens = list(ShiftReduceParser.tokenize_with_lookahead(tokenizer, ''))
    assert len(tokens) == 1
    assert tokens[0] == (BOF_VALUE, None)

    tokens = list(ShiftReduceParser.tokenize_with_lookahead(
        tokenizer, 'a b c'))
    assert len(tokens) == 4
    assert tokens[0] == (BOF_VALUE, 'a')
    assert tokens[1] == ('a', 'b')
    assert tokens[2] == ('b', 'c')
    assert tokens[3] == ('c', None)

示例#4

 def get_default_tokenizer(cls):
     return create_shlex_tokenizer(wordchars=string.ascii_uppercase +
                                   string.ascii_lowercase + string.digits +
                                   '_.', )

示例#5

 def get_default_tokenizer(cls):
     return tokenizers.create_shlex_tokenizer(
         with_bof=True,
         with_eof=True,
         wordchars=string.digits + '.',
     )

示例#6

 def get_default_tokenizer(cls):
     return create_shlex_tokenizer(wordchars=string.digits + '.')

示例#7

 def get_default_tokenizer(cls):
     return tokenizers.create_shlex_tokenizer()