Пример #1
0
def test_tokenstream_expect():
    # expect
    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
    assert s.expect('a') == Token('a', 1)
    assert s.expect('b', 2) == Token('b', 2)
    pytest.raises(AssertionError, s.expect, 'e')
    pytest.raises(AssertionError, s.expect, 'c', 5)
Пример #2
0
def test_tokenstream_expect():
    # expect
    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
    assert s.expect('a') == Token('a', 1)
    assert s.expect('b', 2) == Token('b', 2)
    pytest.raises(AssertionError, s.expect, 'e')
    pytest.raises(AssertionError, s.expect, 'c', 5)
Пример #3
0
def test_tokenstream_debug():
    stream = StringIO.StringIO()

    _original_stdout = sys.stdout
    sys.stdout = stream

    try:
        s = TokenStream(iter((Token('a', 1),)))
        s.debug()
        assert stream.getvalue() == "Token(type='a', value=1)\n"
    finally:
        sys.stdout = _original_stdout
Пример #4
0
def test_tokenstream_debug():
    stream = StringIO.StringIO()

    _original_stdout = sys.stdout
    sys.stdout = stream

    try:
        s = TokenStream(iter((Token('a', 1), )))
        s.debug()
        assert stream.getvalue() == "Token(type='a', value=1)\n"
    finally:
        sys.stdout = _original_stdout
Пример #5
0
def test_tokenstream_test_shift():
    # test
    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
    assert s.test('a')
    s.next()
    assert s.test('b', 2)

    # shift
    assert s.current == Token('b', 2)
    s.shift(Token('f', 5))
    assert s.current == Token('f', 5)
    s.next()
    assert s.current == Token('b', 2)
Пример #6
0
def test_tokenstream_test_shift():
    # test
    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
    assert s.test('a')
    s.next()
    assert s.test('b', 2)

    # shift
    assert s.current == Token('b', 2)
    s.shift(Token('f', 5))
    assert s.current == Token('f', 5)
    s.next()
    assert s.current == Token('b', 2)
Пример #7
0
def test_tokenstream_from_tuple_iter():
    # from_tuple_iter
    s = TokenStream.from_tuple_iter(iter((('a', 1), ('b', 2), ('c', 3))))
    assert s.current == Token('a', 1)

    # iter
    assert isinstance(iter(s), TokenStreamIterator)
    assert tuple(iter(s)) == (Token('a', 1), Token('b', 2), Token('c', 3))
Пример #8
0
def test_tokenstream_from_tuple_iter():
    # from_tuple_iter
    s = TokenStream.from_tuple_iter(iter((('a', 1), ('b', 2), ('c', 3))))
    assert s.current == Token('a', 1)

    # iter
    assert isinstance(iter(s), TokenStreamIterator)
    assert tuple(iter(s)) == (Token('a', 1), Token('b', 2), Token('c', 3))
Пример #9
0
def test_parse_arguments():
    line = '<% gettext("some string", str="foo")'
    stream = TokenStream.from_tuple_iter(tokenize(line, underscore_rules))
    stream.next()
    stream.next()
    stream.expect('gettext_begin')
    funcname = stream.expect('func_name').value
    args, kwargs = parse_arguments(stream, 'gettext_end')
    assert args == ('some string',)
    assert kwargs == {'str': 'foo'}
Пример #10
0
def test_parse_arguments():
    line = '<% gettext("some string", str="foo")'
    stream = TokenStream.from_tuple_iter(tokenize(line, underscore_rules))
    stream.next()
    stream.next()
    stream.expect('gettext_begin')
    funcname = stream.expect('func_name').value
    args, kwargs = parse_arguments(stream, 'gettext_end')
    assert args == ('some string', )
    assert kwargs == {'str': 'foo'}
Пример #11
0
def test_tokenstream_skip_next():
    # skip, next
    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
    s.skip(1)
    assert s.current == Token('b', 2)
    s.next()
    assert s.current == Token('c', 3)
    s.push(Token('e', 4))
    assert s.current == Token('c', 3)
    s.next()
    assert s.current == Token('e', 4)
    s.next()
    assert s.current == Token('eof', None)
Пример #12
0
def extract(fileobj, keywords, comment_tags, options):
    """Extracts translation messages from underscore template files.

    This method does also extract django templates. If a template does not
    contain any django translation tags we always fallback to underscore extraction.

    This is a plugin to Babel, written according to
    http://babel.pocoo.org/docs/messages/#writing-extraction-methods

    :param fileobj: the file-like object the messages should be extracted
                    from
    :param keywords: a list of keywords (i.e. function names) that should
                     be recognized as translation functions
    :param comment_tags: a list of translator tags to search for and
                         include in the results
    :param options: a dictionary of additional options (optional)
    :return: an iterator over ``(lineno, funcname, message, comments)``
             tuples
    :rtype: ``iterator``
    """
    encoding = options.get('encoding', 'utf-8')

    original_position = fileobj.tell()

    text = fileobj.read().decode(encoding)

    if django.VERSION[:2] >= (1, 9):
        tokens = Lexer(text).tokenize()
    else:
        tokens = Lexer(text, None).tokenize()

    vars = [token.token_type != TOKEN_TEXT for token in tokens]

    could_be_django = any(list(vars))

    if could_be_django:
        fileobj.seek(original_position)
        iterator = extract_django(fileobj, keywords, comment_tags, options)
        for lineno, funcname, message, comments in iterator:
            yield lineno, funcname, message, comments
    else:
        # Underscore template extraction
        comments = []

        fileobj.seek(original_position)

        for lineno, line in enumerate(fileobj, 1):
            funcname = None

            stream = TokenStream.from_tuple_iter(
                tokenize(line, underscore.rules))
            while not stream.eof:
                if stream.current.type == 'gettext_begin':
                    stream.expect('gettext_begin')
                    funcname = stream.expect('func_name').value
                    args, kwargs = parse_arguments(stream, 'gettext_end')

                    strings = []

                    for arg, argtype in args:
                        if argtype == 'func_string_arg':
                            strings.append(force_text(arg))
                        else:
                            strings.append(None)

                    for arg in kwargs:
                        strings.append(None)

                    if len(strings) == 1:
                        strings = strings[0]
                    else:
                        strings = tuple(strings)

                    yield lineno, funcname, strings, []

                stream.next()
Пример #13
0
def extract(fileobj, keywords, comment_tags, options):
    """Extracts translation messages from underscore template files.

    This method does also extract django templates. If a template does not
    contain any django translation tags we always fallback to underscore extraction.

    This is a plugin to Babel, written according to
    http://babel.pocoo.org/docs/messages/#writing-extraction-methods

    :param fileobj: the file-like object the messages should be extracted
                    from
    :param keywords: a list of keywords (i.e. function names) that should
                     be recognized as translation functions
    :param comment_tags: a list of translator tags to search for and
                         include in the results
    :param options: a dictionary of additional options (optional)
    :return: an iterator over ``(lineno, funcname, message, comments)``
             tuples
    :rtype: ``iterator``
    """
    encoding = options.get('encoding', 'utf-8')

    original_position = fileobj.tell()

    text = fileobj.read().decode(encoding)

    # TODO: There must be another way. Find a way to fix the ordering
    # in babel directly!
    vars = [token.token_type != TOKEN_TEXT for token in Lexer(text, None).tokenize()]
    could_be_django = any(list(vars))

    if could_be_django:
        fileobj.seek(original_position)
        iterator = extract_django(fileobj, keywords, comment_tags, options)
        for lineno, funcname, message, comments in iterator:
            yield lineno, funcname, message, comments
    else:
        # Underscore template extraction
        comments = []

        fileobj.seek(original_position)

        for lineno, line in enumerate(fileobj, 1):
            funcname = None

            stream = TokenStream.from_tuple_iter(tokenize(line, underscore.rules))
            while not stream.eof:
                if stream.current.type == 'gettext_begin':
                    stream.expect('gettext_begin')
                    funcname = stream.expect('func_name').value
                    args, kwargs = parse_arguments(stream, 'gettext_end')

                    strings = []
                    for arg in args:
                        try:
                            arg = int(arg)
                        except ValueError:
                            pass
                        if isinstance(arg, six.string_types):
                            strings.append(force_text(arg))
                        else:
                            strings.append(None)

                    for arg in kwargs:
                        strings.append(None)

                    if len(strings) == 1:
                        strings = strings[0]
                    else:
                        strings = tuple(strings)

                    yield lineno, funcname, strings, []

                stream.next()
Пример #14
0
def test_tokenstream():
    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
    assert s.current == Token('a', 1)
Пример #15
0
def test_tokenstream_skip_next():
    # skip, next
    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
    s.skip(1)
    assert s.current == Token('b', 2)
    s.next()
    assert s.current == Token('c', 3)
    s.push(Token('e', 4))
    assert s.current == Token('c', 3)
    s.next()
    assert s.current == Token('e', 4)
    s.next()
    assert s.current == Token('eof', None)
Пример #16
0
def test_tokenstream_eof():
    # eof
    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
    assert not s.eof
    list(s)
    assert s.eof
Пример #17
0
def test_tokenstream_look_push():
    # look, push
    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
    assert s.current == Token('a', 1)
    assert s.look() == Token('b', 2)
    s.next()
    assert s.look() == Token('c', 3)
    s.push(Token('b', 2))
    assert s.look() == Token('b', 2)
    s.push(Token('e', 4), current=True)
    assert s.current == Token('e', 4)
    assert s.look() == Token('b', 2)
Пример #18
0
def test_tokenstream_look_push():
    # look, push
    s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
    assert s.current == Token('a', 1)
    assert s.look() == Token('b', 2)
    s.next()
    assert s.look() == Token('c', 3)
    s.push(Token('b', 2))
    assert s.look() == Token('b', 2)
    s.push(Token('e', 4), current=True)
    assert s.current == Token('e', 4)
    assert s.look() == Token('b', 2)