def test_extract_ignored_comment2(self):
buf = BytesIO(
b'{# Translators: ignored i18n comment #1 #}{% trans "Translatable literal #9a" %}'
)
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'Translatable literal #9a', [])],
messages)
def test_blocktrans_with_whitespace_trimmed(self):
test_tmpl = (
b'{% blocktrans trimmed %}\n\tfoo\n\tbar\n{% endblocktrans %}'
)
buf = BytesIO(test_tmpl)
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(4, None, u'foo bar', [])], messages)
def test_extract_valid_comment(self):
buf = BytesIO(
b'{# ignored comment #6 #}{% trans "Translatable literal #9h" %}{# Translators: valid i18n comment #7 #}'
)
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'Translatable literal #9h', [])],
messages)
def test_extract_filter_with_filter(self):
test_tmpl = (
b'{% blocktrans with berta=anton|lower %}'
b'{{ berta }}{% endblocktrans %}'
)
buf = BytesIO(test_tmpl)
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'%(berta)s', [])], messages)
def test_extract_ignored_comment(self):
buf = BytesIO(
b'{# ignored comment #1 #}{% trans "Translatable literal #9a" %}',
)
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual(
[(1, None, u'Translatable literal #9a', [])], messages
)
def test_extract_singular_form(self):
buf = BytesIO(
b'{% blocktrans count counter=number %}singular{% plural %}{{ counter }} plural{% endblocktrans %}'
)
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, 'ngettext',
(u'singular', u'%(counter)s plural'), [])],
messages)
def test_extract_constant_in_block(self):
test_tmpl = (
b'{% blocktrans foo=_("constant") %}{{ foo }}{% endblocktrans %}')
buf = BytesIO(test_tmpl)
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual(
[(1, None, u'"constant"', []), (1, None, u'%(foo)s', [])],
messages,
)
def test_extract_context_in_plural_block(self):
test_tmpl = (b'{% blocktrans context "banana" %}{{ foo }}'
b'{% plural %}{{ bar }}{% endblocktrans %}')
buf = BytesIO(test_tmpl)
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual(
[(1, 'npgettext', [u'banana', u'%(foo)s', u'%(bar)s'], [])],
messages,
)
def test_extract_context_in_block(self):
test_tmpl = (
b'{% blocktrans context "banana" %}{{ foo }}{% endblocktrans %}'
)
buf = BytesIO(test_tmpl)
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual(
[(1, 'pgettext', [u'banana', u'%(foo)s'], [])],
messages,
)
def test_extract_ignored_comment2(self):
test_tmpl = (
b'{# Translators: ignored i18n comment #1 #}'
b'{% trans "Translatable literal #9a" %}'
)
buf = BytesIO(test_tmpl)
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual(
[(1, None, u'Translatable literal #9a', [])], messages
)
def test_extract_constant_in_block(self):
test_tmpl = (
b'{% blocktrans foo=_("constant") %}{{ foo }}{% endblocktrans %}'
)
buf = BytesIO(test_tmpl)
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual(
[(1, None, u'"constant"', []), (1, None, u'%(foo)s', [])],
messages,
)
def test_extract_singular_form(self):
test_tmpl = (
b'{% blocktrans count counter=number %}'
b'singular{% plural %}{{ counter }} plural'
b'{% endblocktrans %}'
)
buf = BytesIO(test_tmpl)
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual(
[(1, 'ngettext', (u'singular', u'%(counter)s plural'), [])],
messages
)
def test_extract_unicode_blocktrans(self):
buf = BytesIO(force_bytes('{% blocktrans %}@ſðæ314“ſſ¶ÐĐÞ→SÆ^ĸŁ{% endblocktrans %}'))
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'@ſðæ314“ſſ¶ÐĐÞ→SÆ^ĸŁ', [])], messages)
def test_trans_blocks_must_not_include_other_block_tags(self):
buf = BytesIO(b'{% blocktrans %}{% other_tag %}{% endblocktrans %}')
gen = extract_django(buf, default_keys, [], {})
pytest.raises(SyntaxError, next, gen)
def test_extract_no_tags(self):
buf = BytesIO(b'nothing')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([], messages)
def extract(fileobj, keywords, comment_tags, options):
"""Extracts translation messages from underscore template files.
This method does also extract django templates. If a template does not
contain any django translation tags we always fallback to underscore extraction.
This is a plugin to Babel, written according to
http://babel.pocoo.org/docs/messages/#writing-extraction-methods
:param fileobj: the file-like object the messages should be extracted
from
:param keywords: a list of keywords (i.e. function names) that should
be recognized as translation functions
:param comment_tags: a list of translator tags to search for and
include in the results
:param options: a dictionary of additional options (optional)
:return: an iterator over ``(lineno, funcname, message, comments)``
tuples
:rtype: ``iterator``
"""
encoding = options.get('encoding', 'utf-8')
original_position = fileobj.tell()
text = fileobj.read().decode(encoding)
# TODO: There must be another way. Find a way to fix the ordering
# in babel directly!
vars = [token.token_type != TOKEN_TEXT for token in Lexer(text, None).tokenize()]
could_be_django = any(list(vars))
if could_be_django:
fileobj.seek(original_position)
iterator = extract_django(fileobj, keywords, comment_tags, options)
for lineno, funcname, message, comments in iterator:
yield lineno, funcname, message, comments
else:
# Underscore template extraction
comments = []
fileobj.seek(original_position)
for lineno, line in enumerate(fileobj, 1):
funcname = None
stream = TokenStream.from_tuple_iter(tokenize(line, underscore.rules))
while not stream.eof:
if stream.current.type == 'gettext_begin':
stream.expect('gettext_begin')
funcname = stream.expect('func_name').value
args, kwargs = parse_arguments(stream, 'gettext_end')
strings = []
for arg in args:
try:
arg = int(arg)
except ValueError:
pass
if isinstance(arg, six.string_types):
strings.append(force_text(arg))
else:
strings.append(None)
for arg in kwargs:
strings.append(None)
if len(strings) == 1:
strings = strings[0]
else:
strings = tuple(strings)
yield lineno, funcname, strings, []
stream.next()
def test_extract_unicode_blocktrans(self):
buf = BytesIO(force_bytes("{% blocktrans %}@ſðæ314“ſſ¶ÐĐÞ→SÆ^ĸŁ{% endblocktrans %}"))
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u"@ſðæ314“ſſ¶ÐĐÞ→SÆ^ĸŁ", [])], messages)
def test_extract_constant_block(self):
buf = BytesIO(b'{% _("constant") %}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'"constant"', [])], messages)
def test_extract_constant_block(self):
buf = BytesIO(b'{% _("constant") %}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'"constant"', [])], messages)
def test_extract_constant_single_quotes(self):
buf = BytesIO(b"{{ _('constant') }}")
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u"'constant'", [])], messages)
def test_extract_valid_comment(self):
buf = BytesIO(b'{# ignored comment #6 #}{% trans "Translatable literal #9h" %}{# Translators: valid i18n comment #7 #}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'Translatable literal #9h', [])], messages)
def test_extract_constant_single_quotes(self):
buf = BytesIO(b'{{ _("constant") }}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'"constant"', [])], messages)
def test_extract_constant_single_quotes(self):
buf = BytesIO(b'{{ _("constant") }}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'"constant"', [])], messages)
def test_extract_unicode_blocktrans(self):
test_tmpl = u'{% blocktrans %}@ſðæ314“ſſ¶ÐĐÞ→SÆ^ĸŁ{% endblocktrans %}'
buf = BytesIO(test_tmpl.encode('utf8'))
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'@ſðæ314“ſſ¶ÐĐÞ→SÆ^ĸŁ', [])], messages)
def test_extract_unicode(self):
buf = BytesIO(force_bytes('{% trans "@ſðæ314“ſſ¶ÐĐÞ→SÆ^ĸŁ" %}'))
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u"@ſðæ314“ſſ¶ÐĐÞ→SÆ^ĸŁ", [])], messages)
def test_extract_var(self):
buf = BytesIO(b'{{ book }}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([], messages)
def test_trans_blocks_must_not_include_other_block_tags(self):
buf = BytesIO(b'{% blocktrans %}{% other_tag %}{% endblocktrans %}')
gen = extract_django(buf, default_keys, [], {})
with pytest.raises(SyntaxError):
next(gen)
def test_extract_filters_default_translatable_single_quotes(self):
buf = BytesIO(b"{{ book.author|default:_('Unknown') }}")
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'Unknown', [])], messages)
def test_extract_filters_default_translatable_single_quotes(self):
buf = BytesIO(b"{{ book.author|default:_('Unknown') }}")
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'Unknown', [])], messages)
def test_extract_no_tags(self):
buf = BytesIO(b'nothing')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([], messages)
def test_extract_unicode(self):
buf = BytesIO(b'{% trans "@ſðæ314“ſſ¶ÐĐÞ→SÆ^ĸŁ" %}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'@ſðæ314“ſſ¶ÐĐÞ→SÆ^ĸŁ', [])], messages)
def test_extract_var_other(self):
buf = BytesIO(b'{{ book }}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([], messages)
def test_extract_with_interpolation(self):
buf = BytesIO(b'{% blocktrans %}xxx{{ anton }}xxx{% endblocktrans %}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'xxx%(anton)sxxx', [])], messages)
def test_extract_constant_single_quotes(self):
buf = BytesIO(b"{{ _('constant') }}")
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u"'constant'", [])], messages)
def test_extract_simple_single_quotes(self):
buf = BytesIO(b"{% trans 'Bunny' %}")
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'Bunny', [])], messages)
def test_extract_var(self):
buf = BytesIO(b'{% blocktrans %}{{ anton }}{% endblocktrans %}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'%(anton)s', [])], messages)
def test_extract_simple_with_context_with_double_quotes(self):
buf = BytesIO(b"{% trans 'Bunny' context '\"carrot\"' %}")
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, 'pgettext', [u'"carrot"', u'Bunny'], [])],
messages)
def test_extract_unicode_blocktrans(self):
test_tmpl = u'{% blocktrans %}@ſðæ314“ſſ¶ÐĐÞ→SÆ^ĸŁ{% endblocktrans %}'
buf = BytesIO(test_tmpl.encode('utf8'))
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'@ſðæ314“ſſ¶ÐĐÞ→SÆ^ĸŁ', [])], messages)
def test_extract_singular_form(self):
buf = BytesIO(
b"{% blocktrans count counter=number %}singular{% plural %}{{ counter }} plural{% endblocktrans %}"
)
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, "ngettext", (u"singular", u"%(counter)s plural"), [])], messages)
def test_extract_simple_with_context_with_double_quotes(self):
buf = BytesIO(b"{% trans 'Bunny' context '\"carrot\"' %}")
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, 'pgettext',
[u'"carrot"', u'Bunny'], [])], messages)
def test_extract_filter_with_filter(self):
buf = BytesIO(b"{% blocktrans with berta=anton|lower %}{{ berta }}{% endblocktrans %}")
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u"%(berta)s", [])], messages)
def test_extract_simple_single_quotes(self):
buf = BytesIO(b"{% trans 'Bunny' %}")
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'Bunny', [])], messages)
def test_extract_filters_default_translatable(self):
buf = BytesIO(b'{{ book.author|default:_("Unknown") }}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u"Unknown", [])], messages)
def test_extract_unicode(self):
buf = BytesIO(u'{% trans "@ſðæ314“ſſ¶ÐĐÞ→SÆ^ĸŁ" %}'.encode('utf8'))
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'@ſðæ314“ſſ¶ÐĐÞ→SÆ^ĸŁ', [])], messages)
def test_extract_var(self):
buf = BytesIO(b'{% blocktrans %}{{ anton }}{% endblocktrans %}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'%(anton)s', [])], messages)
def test_extract_filter_with_filter(self):
buf = BytesIO(b'{% blocktrans with berta=anton|lower %}{{ berta }}{% endblocktrans %}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'%(berta)s', [])], messages)
def test_extract_with_interpolation(self):
buf = BytesIO(b'{% blocktrans %}xxx{{ anton }}xxx{% endblocktrans %}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u'xxx%(anton)sxxx', [])], messages)
def extract(fileobj, keywords, comment_tags, options):
"""Extracts translation messages from underscore template files.
This method does also extract django templates. If a template does not
contain any django translation tags we always fallback to underscore extraction.
This is a plugin to Babel, written according to
http://babel.pocoo.org/docs/messages/#writing-extraction-methods
:param fileobj: the file-like object the messages should be extracted
from
:param keywords: a list of keywords (i.e. function names) that should
be recognized as translation functions
:param comment_tags: a list of translator tags to search for and
include in the results
:param options: a dictionary of additional options (optional)
:return: an iterator over ``(lineno, funcname, message, comments)``
tuples
:rtype: ``iterator``
"""
encoding = options.get('encoding', 'utf-8')
original_position = fileobj.tell()
text = fileobj.read().decode(encoding)
if django.VERSION[:2] >= (1, 9):
tokens = Lexer(text).tokenize()
else:
tokens = Lexer(text, None).tokenize()
vars = [token.token_type != TOKEN_TEXT for token in tokens]
could_be_django = any(list(vars))
if could_be_django:
fileobj.seek(original_position)
iterator = extract_django(fileobj, keywords, comment_tags, options)
for lineno, funcname, message, comments in iterator:
yield lineno, funcname, message, comments
else:
# Underscore template extraction
comments = []
fileobj.seek(original_position)
for lineno, line in enumerate(fileobj, 1):
funcname = None
stream = TokenStream.from_tuple_iter(
tokenize(line, underscore.rules))
while not stream.eof:
if stream.current.type == 'gettext_begin':
stream.expect('gettext_begin')
funcname = stream.expect('func_name').value
args, kwargs = parse_arguments(stream, 'gettext_end')
strings = []
for arg, argtype in args:
if argtype == 'func_string_arg':
strings.append(force_text(arg))
else:
strings.append(None)
for arg in kwargs:
strings.append(None)
if len(strings) == 1:
strings = strings[0]
else:
strings = tuple(strings)
yield lineno, funcname, strings, []
stream.next()
def test_extract_simple_double_quotes(self):
buf = BytesIO(b'{% trans "Bunny" %}')
messages = list(extract_django(buf, default_keys, [], {}))
self.assertEqual([(1, None, u"Bunny", [])], messages)