def extract_tower_template(fileobj, keywords, comment_tags, options):
for lineno, funcname, message, comments in \
list(ext.babel_extract(fileobj, keywords, comment_tags, options)):
message = tweak_message(message)
yield lineno, funcname, message, comments
def extract_jade(fileobj, keywords, comment_tags, options):
"""
Extract messages from .jade files.
NOTE: Line numbers are currently broken
:param fileobj: the file-like object the messages should be extracted
from
:param keywords: a list of keywords (i.e. function names) that should
be recognized as translation functions
:param comment_tags: a list of translator tags to search for and
include in the results
:param options: a dictionary of additional options (optional)
:return: an iterator over ``(lineno, funcname, message, comments)``
tuples
:rtype: ``iterator``
"""
src_jade = six.b("").join(fileobj.readlines())
parser = pyjade.parser.Parser(src_jade)
block = parser.parse()
compiler = JinjaCompiler(block, pretty=True)
src_jinja = compiler.compile()
new_fileobj = six.BytesIO(bytes(src_jinja, 'UTF-8'))
def drop_lineno(v):
return (0, v[1], v[2], v[3])
return map(drop_lineno, babel_extract(new_fileobj, keywords, comment_tags, options))
def extract_tower_template(fileobj, keywords, comment_tags, options):
for lineno, funcname, message, comments in \
list(ext.babel_extract(fileobj, keywords, comment_tags, options)):
message = tweak_message(message)
yield lineno, funcname, message, comments
def jinja2_babel_extract(fileobj, keywords, comment_tags, options):
"""Custom babel_extract for Jinja.
Hooks on to Jinja's babel_extract and handles
whitespace within ``{% trans %}`` tags.
"""
for lineno, func, message, comments in babel_extract(
fileobj, keywords, comment_tags, options):
yield lineno, func, trim_inner_whitespace(message), comments
def _get_nereid_template_messages_from_file(self, template_dir, template):
"""
Same generator as _get_nereid_template_messages, but for specific files.
"""
extract_options = self._get_nereid_template_extract_options()
loader = FileSystemLoader(template_dir)
file_obj = open(loader.get_source({}, template)[1])
for message_tuple in babel_extract(file_obj, GETTEXT_FUNCTIONS,
['trans:'], extract_options):
yield (template, ) + message_tuple
def _get_nereid_template_messages_from_file(self, template_dir, template):
"""
Same generator as _get_nereid_template_messages, but for specific files.
"""
extract_options = self._get_nereid_template_extract_options()
loader = FileSystemLoader(template_dir)
file_obj = open(loader.get_source({}, template)[1])
for message_tuple in babel_extract(
file_obj, GETTEXT_FUNCTIONS,
['trans:'], extract_options):
yield (template,) + message_tuple
def test_extract(self):
from jinja2.ext import babel_extract
source = BytesIO('''
{{ gettext('Hello World') }}
{% trans %}Hello World{% endtrans %}
{% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
'''.encode('ascii')) # make python 3 happy
assert list(babel_extract(source, ('gettext', 'ngettext', '_'), [], {})) == [
(2, 'gettext', u'Hello World', []),
(3, 'gettext', u'Hello World', []),
(4, 'ngettext', (u'%(users)s user', u'%(users)s users', None), [])
]
def jinja2_babel_extract(fileobj, keywords, comment_tags, options):
"""Custom babel_extract for Jinja.
Hooks on to Jinja's babel_extract and handles
whitespace within ``{% trans %}`` tags.
"""
for lineno, func, message, comments in babel_extract(fileobj, keywords, comment_tags, options):
if isinstance(message, tuple):
message = tuple(trim_inner_whitespace(x) if isinstance(x, basestring) else x for x in message)
else:
message = trim_inner_whitespace(message)
yield lineno, func, message, comments
def jinja2_babel_extract(fileobj, keywords, comment_tags, options):
"""Custom babel_extract for Jinja.
Hooks on to Jinja's babel_extract and handles
whitespace within ``{% trans %}`` tags.
"""
for lineno, func, message, comments in babel_extract(fileobj, keywords, comment_tags, options):
if isinstance(message, tuple):
message = tuple(trim_inner_whitespace(x) if isinstance(x, basestring) else x for x in message)
else:
message = trim_inner_whitespace(message)
yield lineno, func, message, comments
def test_extract(self):
from jinja2.ext import babel_extract
source = BytesIO('''
{{ gettext('Hello World') }}
{% trans %}Hello World{% endtrans %}
{% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
'''.encode('ascii')) # make python 3 happy
assert list(babel_extract(source, ('gettext', 'ngettext', '_'), [], {})) == [
(2, 'gettext', u'Hello World', []),
(3, 'gettext', u'Hello World', []),
(4, 'ngettext', (u'%(users)s user', u'%(users)s users', None), [])
]
def test_extract_context(self):
from jinja2.ext import babel_extract
source = BytesIO(b"""
{{ pgettext("babel", "Hello World") }}
{{ npgettext("babel", "%(users)s user", "%(users)s users", users) }}
""")
assert list(
babel_extract(source, ("pgettext", "npgettext", "_"), [], {})) == [
(2, "pgettext", ("babel", "Hello World"), []),
(3, "npgettext", ("babel", "%(users)s user", "%(users)s users",
None), []),
]
def test_extract():
from jinja2.ext import babel_extract
from StringIO import StringIO
source = StringIO('''
{{ gettext('Hello World') }}
{% trans %}Hello World{% endtrans %}
{% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
''')
assert list(babel_extract(source, ('gettext', 'ngettext', '_'), [], {})) == [
(2, 'gettext', u'Hello World', []),
(3, 'gettext', u'Hello World', []),
(4, 'ngettext', (u'%(users)s user', u'%(users)s users', None), [])
]
def test_comment_extract(self):
from jinja2.ext import babel_extract
source = BytesIO('''
{# trans first #}
{{ gettext('Hello World') }}
{% trans %}Hello World{% endtrans %}{# trans second #}
{#: third #}
{% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
'''.encode('utf-8')) # make python 3 happy
assert list(babel_extract(source, ('gettext', 'ngettext', '_'), ['trans', ':'], {})) == [
(3, 'gettext', u'Hello World', ['first']),
(4, 'gettext', u'Hello World', ['second']),
(6, 'ngettext', (u'%(users)s user', u'%(users)s users', None), ['third'])
]
def test_extract():
from jinja2.ext import babel_extract
from StringIO import StringIO
source = StringIO('''
{{ gettext('Hello World') }}
{% trans %}Hello World{% endtrans %}
{% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
''')
assert list(babel_extract(source, ('gettext', 'ngettext', '_'), [],
{})) == [(2, 'gettext', u'Hello World', []),
(3, 'gettext', u'Hello World', []),
(4, 'ngettext',
(u'%(users)s user', u'%(users)s users',
None), [])]
def test_comment_extract(self):
from jinja2.ext import babel_extract
source = BytesIO('''
{# trans first #}
{{ gettext('Hello World') }}
{% trans %}Hello World{% endtrans %}{# trans second #}
{#: third #}
{% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
'''.encode('utf-8')) # make python 3 happy
assert list(babel_extract(source, ('gettext', 'ngettext', '_'), ['trans', ':'], {})) == [
(3, 'gettext', u'Hello World', ['first']),
(4, 'gettext', u'Hello World', ['second']),
(6, 'ngettext', (u'%(users)s user', u'%(users)s users', None), ['third'])
]
def test_extract(self):
from jinja2.ext import babel_extract
source = BytesIO(
b"""
{{ gettext('Hello World') }}
{% trans %}Hello World{% endtrans %}
{% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
"""
)
assert list(babel_extract(source, ("gettext", "ngettext", "_"), [], {})) == [
(2, "gettext", "Hello World", []),
(3, "gettext", "Hello World", []),
(4, "ngettext", ("%(users)s user", "%(users)s users", None), []),
]
def test_extract_trimmed_option(self):
from jinja2.ext import babel_extract
source = BytesIO('''
{{ gettext(' Hello \n World') }}
{% trans %} Hello \n World{% endtrans %}
{% trans %}{{ users }} \n user
{%- pluralize %}{{ users }} \n users{% endtrans %}
'''.encode('ascii')) # make python 3 happy
opts = {'trimmed': 'true'}
assert list(babel_extract(source,
('gettext', 'ngettext', '_'), [], opts)) == [
(2, 'gettext', u' Hello \n World', []),
(4, 'gettext', u'Hello World', []),
(6, 'ngettext', (u'%(users)s user', u'%(users)s users', None), [])
]
def extract_ckan(fileobj, *args, **kw):
extensions = [
':'.join([ext.__module__, ext.__name__])
if isinstance(ext, type) else ext for ext in _get_extensions()
]
if 'options' not in kw:
kw['options'] = {}
if 'trimmed' not in kw['options']:
kw['options']['trimmed'] = 'True'
if 'silent' not in kw['options']:
kw['options']['silent'] = 'False'
if 'extensions' not in kw['options']:
kw['options']['extensions'] = ','.join(extensions)
return babel_extract(fileobj, *args, **kw)
def test_extract(self):
from jinja2.ext import babel_extract
source = BytesIO("""
{{ gettext('Hello World') }}
{% trans %}Hello World{% endtrans %}
{% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
""".encode("ascii")) # make python 3 happy
assert list(babel_extract(source, ("gettext", "ngettext", "_"), [],
{})) == [
(2, "gettext", u"Hello World", []),
(3, "gettext", u"Hello World", []),
(4, "ngettext",
(u"%(users)s user", u"%(users)s users",
None), []),
]
def test_extract_trimmed_option(self):
from jinja2.ext import babel_extract
source = BytesIO("""
{{ gettext(' Hello \n World') }}
{% trans %} Hello \n World{% endtrans %}
{% trans %}{{ users }} \n user
{%- pluralize %}{{ users }} \n users{% endtrans %}
""".encode("ascii")) # make python 3 happy
opts = {"trimmed": "true"}
assert list(
babel_extract(source, ("gettext", "ngettext", "_"), [], opts)) == [
(2, "gettext", u" Hello \n World", []),
(4, "gettext", u"Hello World", []),
(6, "ngettext", (u"%(users)s user", u"%(users)s users", None),
[]),
]
def test_extract(self):
from jinja2.ext import babel_extract
source = BytesIO(
"""
{{ gettext('Hello World') }}
{% trans %}Hello World{% endtrans %}
{% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
""".encode(
"ascii"
)
) # make python 3 happy
assert list(babel_extract(source, ("gettext", "ngettext", "_"), [], {})) == [
(2, "gettext", u"Hello World", []),
(3, "gettext", u"Hello World", []),
(4, "ngettext", (u"%(users)s user", u"%(users)s users", None), []),
]
def _get_nereid_template_messages(cls):
"""
Extract localizable strings from the templates of installed modules.
For every string found this function yields a
`(module, template, lineno, function, message)` tuple, where:
* module is the name of the module in which the template is found
* template is the name of the template in which message was found
* lineno is the number of the line on which the string was found,
* function is the name of the gettext function used (if the string
was extracted from embedded Python code), and
* message is the string itself (a unicode object, or a tuple of
unicode objects for functions with multiple string arguments).
* comments List of Translation comments if any. Comments in the code
should have a prefix `trans:`. Example::
{{ _(Welcome) }} {# trans: In the top banner #}
"""
extract_options = cls._get_nereid_template_extract_options()
logger = logging.getLogger('nereid.translation')
for module, directory in cls._get_installed_module_directories():
template_dir = os.path.join(directory, 'templates')
if not os.path.isdir(template_dir):
# The template directory does not exist. Just continue
continue
logger.info(
'Found template directory for module %s at %s' % (
module, template_dir
)
)
# now that there is a template directory, load the templates
# using a simple filesystem loader and load all the
# translations from it.
loader = FileSystemLoader(template_dir)
env = Environment(loader=loader)
extensions = '.html,.jinja'
for template in env.list_templates(extensions=extensions):
logger.info('Loading from: %s:%s' % (module, template))
file_obj = open(loader.get_source({}, template)[1])
for message_tuple in babel_extract(
file_obj, GETTEXT_FUNCTIONS,
['trans:'], extract_options):
yield (module, template) + message_tuple
def test_comment_extract(self):
from jinja2.ext import babel_extract
source = BytesIO("""
{# trans first #}
{{ gettext('Hello World') }}
{% trans %}Hello World{% endtrans %}{# trans second #}
{#: third #}
{% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
""".encode("utf-8")) # make python 3 happy
assert list(
babel_extract(source, ("gettext", "ngettext", "_"), ["trans", ":"],
{})) == [
(3, "gettext", u"Hello World", ["first"]),
(4, "gettext", u"Hello World", ["second"]),
(6, "ngettext", (u"%(users)s user",
u"%(users)s users", None),
["third"]),
]
def test_comment_extract(self):
from jinja2.ext import babel_extract
source = BytesIO(
"""
{# trans first #}
{{ gettext('Hello World') }}
{% trans %}Hello World{% endtrans %}{# trans second #}
{#: third #}
{% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
""".encode(
"utf-8"
)
) # make python 3 happy
assert list(babel_extract(source, ("gettext", "ngettext", "_"), ["trans", ":"], {})) == [
(3, "gettext", u"Hello World", ["first"]),
(4, "gettext", u"Hello World", ["second"]),
(6, "ngettext", (u"%(users)s user", u"%(users)s users", None), ["third"]),
]
def extract_from_jinja2(*args, **kwargs):
"""Just like Jinja2's Babel extractor, but fixes the i18n issue
The Jinja2 Babel extractor appends the i18n extension to the list of
extensions before extracting. Since Puente has its own i18n extension, this
creates problems. So we monkeypatch and then call Jinja2's Babel extractor.
.. Note::
You only need to use this if you're using Babel's pybabel extract. If
you're using Puente's extract command, then it does this already and you
can use the Jinja2 Babel extractor.
"""
# Must monkeypatch first to fix InternationalizationExtension
# stomping issues! See docstring for details.
monkeypatch_i18n()
return babel_extract(*args, **kwargs)
def _add(self, reader, filepath=None):
# Parse messages to translate
extracted = list(babel_extract(reader, ('gettext', 'ngettext', '_'), [], {}))
# Add messages to translate to catalog
self.add_messages(extracted, filepath=filepath)
