def test_tokenstream_expect():
# expect
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert s.expect('a') == Token('a', 1)
assert s.expect('b', 2) == Token('b', 2)
pytest.raises(AssertionError, s.expect, 'e')
pytest.raises(AssertionError, s.expect, 'c', 5)
def test_tokenstream_expect():
# expect
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert s.expect('a') == Token('a', 1)
assert s.expect('b', 2) == Token('b', 2)
pytest.raises(AssertionError, s.expect, 'e')
pytest.raises(AssertionError, s.expect, 'c', 5)
def test_tokenstream_debug():
stream = StringIO.StringIO()
_original_stdout = sys.stdout
sys.stdout = stream
try:
s = TokenStream(iter((Token('a', 1),)))
s.debug()
assert stream.getvalue() == "Token(type='a', value=1)\n"
finally:
sys.stdout = _original_stdout
def test_tokenstream_debug():
stream = StringIO.StringIO()
_original_stdout = sys.stdout
sys.stdout = stream
try:
s = TokenStream(iter((Token('a', 1), )))
s.debug()
assert stream.getvalue() == "Token(type='a', value=1)\n"
finally:
sys.stdout = _original_stdout
def test_tokenstream_test_shift():
# test
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert s.test('a')
s.next()
assert s.test('b', 2)
# shift
assert s.current == Token('b', 2)
s.shift(Token('f', 5))
assert s.current == Token('f', 5)
s.next()
assert s.current == Token('b', 2)
def test_tokenstream_test_shift():
# test
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert s.test('a')
s.next()
assert s.test('b', 2)
# shift
assert s.current == Token('b', 2)
s.shift(Token('f', 5))
assert s.current == Token('f', 5)
s.next()
assert s.current == Token('b', 2)
def test_tokenstream_from_tuple_iter():
# from_tuple_iter
s = TokenStream.from_tuple_iter(iter((('a', 1), ('b', 2), ('c', 3))))
assert s.current == Token('a', 1)
# iter
assert isinstance(iter(s), TokenStreamIterator)
assert tuple(iter(s)) == (Token('a', 1), Token('b', 2), Token('c', 3))
def test_tokenstream_from_tuple_iter():
# from_tuple_iter
s = TokenStream.from_tuple_iter(iter((('a', 1), ('b', 2), ('c', 3))))
assert s.current == Token('a', 1)
# iter
assert isinstance(iter(s), TokenStreamIterator)
assert tuple(iter(s)) == (Token('a', 1), Token('b', 2), Token('c', 3))
def test_parse_arguments():
line = '<% gettext("some string", str="foo")'
stream = TokenStream.from_tuple_iter(tokenize(line, underscore_rules))
stream.next()
stream.next()
stream.expect('gettext_begin')
funcname = stream.expect('func_name').value
args, kwargs = parse_arguments(stream, 'gettext_end')
assert args == ('some string',)
assert kwargs == {'str': 'foo'}
def test_parse_arguments():
line = '<% gettext("some string", str="foo")'
stream = TokenStream.from_tuple_iter(tokenize(line, underscore_rules))
stream.next()
stream.next()
stream.expect('gettext_begin')
funcname = stream.expect('func_name').value
args, kwargs = parse_arguments(stream, 'gettext_end')
assert args == ('some string', )
assert kwargs == {'str': 'foo'}
def test_tokenstream_skip_next():
# skip, next
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
s.skip(1)
assert s.current == Token('b', 2)
s.next()
assert s.current == Token('c', 3)
s.push(Token('e', 4))
assert s.current == Token('c', 3)
s.next()
assert s.current == Token('e', 4)
s.next()
assert s.current == Token('eof', None)
def extract(fileobj, keywords, comment_tags, options):
"""Extracts translation messages from underscore template files.
This method does also extract django templates. If a template does not
contain any django translation tags we always fallback to underscore extraction.
This is a plugin to Babel, written according to
http://babel.pocoo.org/docs/messages/#writing-extraction-methods
:param fileobj: the file-like object the messages should be extracted
from
:param keywords: a list of keywords (i.e. function names) that should
be recognized as translation functions
:param comment_tags: a list of translator tags to search for and
include in the results
:param options: a dictionary of additional options (optional)
:return: an iterator over ``(lineno, funcname, message, comments)``
tuples
:rtype: ``iterator``
"""
encoding = options.get('encoding', 'utf-8')
original_position = fileobj.tell()
text = fileobj.read().decode(encoding)
if django.VERSION[:2] >= (1, 9):
tokens = Lexer(text).tokenize()
else:
tokens = Lexer(text, None).tokenize()
vars = [token.token_type != TOKEN_TEXT for token in tokens]
could_be_django = any(list(vars))
if could_be_django:
fileobj.seek(original_position)
iterator = extract_django(fileobj, keywords, comment_tags, options)
for lineno, funcname, message, comments in iterator:
yield lineno, funcname, message, comments
else:
# Underscore template extraction
comments = []
fileobj.seek(original_position)
for lineno, line in enumerate(fileobj, 1):
funcname = None
stream = TokenStream.from_tuple_iter(
tokenize(line, underscore.rules))
while not stream.eof:
if stream.current.type == 'gettext_begin':
stream.expect('gettext_begin')
funcname = stream.expect('func_name').value
args, kwargs = parse_arguments(stream, 'gettext_end')
strings = []
for arg, argtype in args:
if argtype == 'func_string_arg':
strings.append(force_text(arg))
else:
strings.append(None)
for arg in kwargs:
strings.append(None)
if len(strings) == 1:
strings = strings[0]
else:
strings = tuple(strings)
yield lineno, funcname, strings, []
stream.next()
def extract(fileobj, keywords, comment_tags, options):
"""Extracts translation messages from underscore template files.
This method does also extract django templates. If a template does not
contain any django translation tags we always fallback to underscore extraction.
This is a plugin to Babel, written according to
http://babel.pocoo.org/docs/messages/#writing-extraction-methods
:param fileobj: the file-like object the messages should be extracted
from
:param keywords: a list of keywords (i.e. function names) that should
be recognized as translation functions
:param comment_tags: a list of translator tags to search for and
include in the results
:param options: a dictionary of additional options (optional)
:return: an iterator over ``(lineno, funcname, message, comments)``
tuples
:rtype: ``iterator``
"""
encoding = options.get('encoding', 'utf-8')
original_position = fileobj.tell()
text = fileobj.read().decode(encoding)
# TODO: There must be another way. Find a way to fix the ordering
# in babel directly!
vars = [token.token_type != TOKEN_TEXT for token in Lexer(text, None).tokenize()]
could_be_django = any(list(vars))
if could_be_django:
fileobj.seek(original_position)
iterator = extract_django(fileobj, keywords, comment_tags, options)
for lineno, funcname, message, comments in iterator:
yield lineno, funcname, message, comments
else:
# Underscore template extraction
comments = []
fileobj.seek(original_position)
for lineno, line in enumerate(fileobj, 1):
funcname = None
stream = TokenStream.from_tuple_iter(tokenize(line, underscore.rules))
while not stream.eof:
if stream.current.type == 'gettext_begin':
stream.expect('gettext_begin')
funcname = stream.expect('func_name').value
args, kwargs = parse_arguments(stream, 'gettext_end')
strings = []
for arg in args:
try:
arg = int(arg)
except ValueError:
pass
if isinstance(arg, six.string_types):
strings.append(force_text(arg))
else:
strings.append(None)
for arg in kwargs:
strings.append(None)
if len(strings) == 1:
strings = strings[0]
else:
strings = tuple(strings)
yield lineno, funcname, strings, []
stream.next()
def test_tokenstream():
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert s.current == Token('a', 1)
def test_tokenstream_skip_next():
# skip, next
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
s.skip(1)
assert s.current == Token('b', 2)
s.next()
assert s.current == Token('c', 3)
s.push(Token('e', 4))
assert s.current == Token('c', 3)
s.next()
assert s.current == Token('e', 4)
s.next()
assert s.current == Token('eof', None)
def test_tokenstream_eof():
# eof
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert not s.eof
list(s)
assert s.eof
def test_tokenstream_look_push():
# look, push
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert s.current == Token('a', 1)
assert s.look() == Token('b', 2)
s.next()
assert s.look() == Token('c', 3)
s.push(Token('b', 2))
assert s.look() == Token('b', 2)
s.push(Token('e', 4), current=True)
assert s.current == Token('e', 4)
assert s.look() == Token('b', 2)
def test_tokenstream_look_push():
# look, push
s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3))))
assert s.current == Token('a', 1)
assert s.look() == Token('b', 2)
s.next()
assert s.look() == Token('c', 3)
s.push(Token('b', 2))
assert s.look() == Token('b', 2)
s.push(Token('e', 4), current=True)
assert s.current == Token('e', 4)
assert s.look() == Token('b', 2)
