def extract_vue(fileobj, keywords, comment_tags, options):
"""Extract messages from Vue template files.
:param fileobj: the file-like the messages should be extracted from
:param keywords: a list of keywords (i.e. function names) that should be recognize as translation functions
:param comment_tags: a list of translator tags to search for and include in the results
:param options: a dictionary of additional options (optional)
:return: an iterator over ``(lineno, funcname, message, comments)``
:rtype: ``iterator``
"""
encoding = options.get('encoding', 'utf-8')
contents = fileobj.read().decode(encoding=encoding)
lexer = Lexer(contents, None)
for t in lexer.tokenize(): # type: Token
if t.token_type in TOKENS:
for i in extract_javascript(
BytesIO(t.contents.encode(encoding=encoding)), keywords,
comment_tags, options):
if i:
yield (t.lineno, i[1], i[2], i[3])
# Inline JS part
matchjs = re.search(r'<script>(.+)</script>', contents, re.DOTALL)
if not matchjs:
return
jscontent = matchjs.group(1)
skipped_line_count = contents[:matchjs.start(1)].count('\n')
for i in extract_javascript(BytesIO(jscontent.encode(encoding=encoding)),
keywords, comment_tags, options):
yield (skipped_line_count + i[0], i[1], i[2], i[3])
def test_jsx_extraction(jsx_enabled):
buf = BytesIO(JSX_SOURCE)
messages = [m[2] for m in extract.extract_javascript(buf, ('_', 'gettext'), [], {"jsx": jsx_enabled})]
if jsx_enabled:
assert messages == EXPECTED_JSX_MESSAGES
else:
assert messages != EXPECTED_JSX_MESSAGES
def test_misplaced_comments(self):
buf = BytesIO(
b"""\
/* NOTE: this won't show up */
foo()
/* NOTE: this will */
msg = _('Something')
// NOTE: this will show up
// too.
msg = _('Something else')
// NOTE: but this won't
bar()
_('no comment here')
"""
)
messages = list(extract.extract_javascript(buf, ("_",), ["NOTE:"], {}))
self.assertEqual(u"Something", messages[0][2])
self.assertEqual([u"NOTE: this will"], messages[0][3])
self.assertEqual(u"Something else", messages[1][2])
self.assertEqual([u"NOTE: this will show up", "too."], messages[1][3])
self.assertEqual(u"no comment here", messages[2][2])
self.assertEqual([], messages[2][3])
def test_misplaced_comments(self):
buf = StringIO("""\
/* NOTE: this won't show up */
foo()
/* NOTE: this will */
msg = _('Something')
// NOTE: this will show up
// too.
msg = _('Something else')
// NOTE: but this won't
bar()
_('no comment here')
""")
messages = list(extract.extract_javascript(buf, ('_', ), ['NOTE:'],
{}))
self.assertEqual(u'Something', messages[0][2])
self.assertEqual([u'NOTE: this will'], messages[0][3])
self.assertEqual(u'Something else', messages[1][2])
self.assertEqual([u'NOTE: this will show up', 'too.'], messages[1][3])
self.assertEqual(u'no comment here', messages[2][2])
self.assertEqual([], messages[2][3])
def test_ignore_function_definitions():
buf = BytesIO(b"""\
function gettext(value) {
return translations[language][value] || value;
}""")
messages = list(extract.extract_javascript(buf, ('gettext',), [], {}))
assert not messages
def test_message_with_line_comment(self):
buf = BytesIO(u"""\
// NOTE: hello
msg = _('Bonjour à tous')
""".encode('utf-8'))
messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
self.assertEqual(u'Bonjour à tous', messages[0][2])
self.assertEqual([u'NOTE: hello'], messages[0][3])
def test_message_with_line_comment(self):
buf = StringIO("""\
// NOTE: hello
msg = _('Bonjour à tous')
""")
messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
self.assertEqual('Bonjour \xe0 tous', messages[0][2])
self.assertEqual(['NOTE: hello'], messages[0][3])
def test_ignore_function_definitions(self):
buf = StringIO("""\
function gettext(value) {
return translations[language][value] || value;
}""")
messages = list(extract.extract_javascript(buf, ('gettext',), [], {}))
self.assertEqual(messages, [])
def test_ignore_function_definitions(self):
buf = StringIO("""\
function gettext(value) {
return translations[language][value] || value;
}""")
messages = list(extract.extract_javascript(buf, ('gettext', ), [], {}))
self.assertEqual(messages, [])
def test_message_with_line_comment():
buf = BytesIO(u"""\
// NOTE: hello
msg = _('Bonjour à tous')
""".encode('utf-8'))
messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
assert messages[0][2] == u'Bonjour à tous'
assert messages[0][3] == [u'NOTE: hello']
def test_message_with_line_comment():
buf = BytesIO(u"""\
// NOTE: hello
msg = _('Bonjour à tous')
""".encode('utf-8'))
messages = list(extract.extract_javascript(buf, ('_', ), ['NOTE:'], {}))
assert messages[0][2] == u'Bonjour à tous'
assert messages[0][3] == [u'NOTE: hello']
def extract_javascript_msgids(source):
"""Return message ids of translateable strings in JS source."""
extracted = extract_javascript(
fileobj=StringIO(source), keywords={"_": None, "P_": (1, 2), "N_": None}, comment_tags={}, options={}
)
return [msg_id for line, func, msg_id, comments in extracted]
def test_ignore_function_definitions():
buf = BytesIO(b"""\
function gettext(value) {
return translations[language][value] || value;
}""")
messages = list(extract.extract_javascript(buf, ('gettext', ), [], {}))
assert not messages
def test_message_with_multiple_line_comments(self):
buf = StringIO("""\
// NOTE: hello
// goodbye
msg = _('Bonjour à tous')
""")
messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
self.assertEqual(u'Bonjour à tous', messages[0][2])
self.assertEqual([u'NOTE: hello goodbye'], messages[0][3])
def test_message_with_line_comment(self):
buf = StringIO("""\
// NOTE: hello
msg = _('Bonjour à tous')
""")
messages = list(extract.extract_javascript(buf, ('_', ), ['NOTE:'],
{}))
self.assertEqual(u'Bonjour à tous', messages[0][2])
self.assertEqual([u'NOTE: hello'], messages[0][3])
def test_jsx_extraction(jsx_enabled):
buf = BytesIO(JSX_SOURCE)
messages = [
m[2] for m in extract.extract_javascript(buf, (
'_', 'gettext'), [], {"jsx": jsx_enabled})
]
if jsx_enabled:
assert messages == EXPECTED_JSX_MESSAGES
else:
assert messages != EXPECTED_JSX_MESSAGES
def test_message_with_multiline_comment(self):
buf = StringIO("""\
/* NOTE: hello
and bonjour
and servus */
msg = _('Bonjour à tous')
""")
messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
self.assertEqual(u'Bonjour à tous', messages[0][2])
self.assertEqual([u'NOTE: hello', 'and bonjour', ' and servus'], messages[0][3])
def test_message_with_multiline_comment(self):
buf = BytesIO(u"""\
/* NOTE: hello
and bonjour
and servus */
msg = _('Bonjour à tous')
""".encode('utf-8'))
messages = list(extract.extract_javascript(buf, ('_', ), ['NOTE:'],
{}))
self.assertEqual(u'Bonjour à tous', messages[0][2])
self.assertEqual([u'NOTE: hello', 'and bonjour', ' and servus'],
messages[0][3])
def test_message_with_line_comment(self):
buf = BytesIO(
u"""\
// NOTE: hello
msg = _('Bonjour à tous')
""".encode(
"utf-8"
)
)
messages = list(extract.extract_javascript(buf, ("_",), ["NOTE:"], {}))
self.assertEqual(u"Bonjour à tous", messages[0][2])
self.assertEqual([u"NOTE: hello"], messages[0][3])
def test_message_with_multiline_comment(self):
buf = StringIO("""\
/* NOTE: hello
and bonjour
and servus */
msg = _('Bonjour à tous')
""")
messages = list(extract.extract_javascript(buf, ('_', ), ['NOTE:'],
{}))
self.assertEqual(u('Bonjour \xe0 tous'), messages[0][2])
self.assertEqual([u('NOTE: hello'), 'and bonjour', ' and servus'],
messages[0][3])
def extract_javascript_script(fileobj, keywords, comment_tags, options):
"""Extract messages from Javascript embedding in <script> tags.
Select <script type="javascript/text"> tags and delegate to
`extract_javascript`.
"""
from genshi.core import Stream
from genshi.input import XMLParser
out = StringIO()
stream = Stream(XMLParser(fileobj))
stream.select('//script[@type="text/javascript"]').render(out=out)
out.seek(0)
return extract_javascript(out, keywords, comment_tags, options)
def extract_javascript_script(fileobj, keywords, comment_tags, options):
"""Extract messages from Javascript embedded in <script> tags.
Select <script type="javascript/text"> tags and delegate to
`extract_javascript`.
"""
if not fileobj.name:
return []
out = io.StringIO()
extractor = ScriptExtractor(out)
extractor.feed(unicode(fileobj.read(), 'utf-8'))
extractor.close()
out.seek(0)
return extract_javascript(out, keywords, comment_tags, options)
def extract_javascript_msgids(source):
"""Return message ids of translateable strings in JS source."""
extracted = extract_javascript(
fileobj=StringIO(source),
keywords={
"_": None,
"P_": (1, 2),
"N_": None,
},
comment_tags={},
options={},
)
return [msg_id for line, func, msg_id, comments in extracted]
def extract_mxml(fileobj, keywords, comment_tags, options):
for elem, pos in MXMLParser(fileobj).parse():
if elem.tag == 'mx:Script':
for lineno, funcname, message, comments in \
extract_javascript(StringIO(elem.text), keywords,
comment_tags, options):
yield pos[0]+lineno, funcname, message, comments
else:
attrib = None
for attr in options.get('attrs', []):
if elem.get(attr):
attrib = attr
if attrib:
if elem.attrib[attrib].startswith('{') and \
elem.attrib[attrib].endswith('}'):
for _, funcname, message, comments in extract_actionscript(
StringIO(elem.attrib[attrib][1:-1]),
keywords, comment_tags, options):
yield pos[0], funcname, message, comments
def extract_vue(fileobj, keywords, comment_tags, options):
"""Extract messages from Vue template files.
:param fileobj: the file-like the messages should be extracted from
:param keywords: a list of keywords (i.e. function names) that should be recognize as translation functions
:param comment_tags: a list of translator tags to search for and include in the results
:param options: a dictionary of additional options (optional)
:return: an iterator over ``(lineno, funcname, message, comments)``
:rtype: ``iterator``
"""
encoding = options.get('encoding', 'utf-8')
contents = fileobj.read().decode(encoding=encoding)
lexer = Lexer(contents, None)
for t in lexer.tokenize(): # type: Token
if t.token_type in TOKENS:
for i in extract_javascript(
BytesIO(t.contents.encode(encoding=encoding)), keywords,
comment_tags, options):
if i:
yield (t.lineno, i[1], i[2], i[3])
def extract_javascript(fileobj, keywords, comment_tags, options):
"""Extract messages from Javascript source files. This extractor delegates
to babel's buit-in javascript extractor, but adds a special comment
used as a flag to identify web translations.
:param fileobj: the file-like object the messages should be extracted
from
:param keywords: a list of keywords (i.e. function names) that should
be recognized as translation functions
:param comment_tags: a list of translator tags to search for and
include in the results
:param options: a dictionary of additional options (optional)
:return: an iterator over ``(lineno, funcname, message, comments)``
tuples
:rtype: ``iterator``
"""
for (message_lineno, funcname, messages, comments) in \
extract.extract_javascript(fileobj, keywords, comment_tags, options):
comments.append(TRANSLATION_FLAG_COMMENT)
yield (message_lineno, funcname, messages, comments)
def extract_javascript(fileobj, keywords, comment_tags, options):
"""Extract messages from Javascript source files. This extractor delegates
to babel's buit-in javascript extractor, but adds a special comment
used as a flag to identify web translations.
:param fileobj: the file-like object the messages should be extracted
from
:param keywords: a list of keywords (i.e. function names) that should
be recognized as translation functions
:param comment_tags: a list of translator tags to search for and
include in the results
:param options: a dictionary of additional options (optional)
:return: an iterator over ``(lineno, funcname, message, comments)``
tuples
:rtype: ``iterator``
"""
for (message_lineno, funcname, messages, comments) in \
extract.extract_javascript(fileobj, keywords, comment_tags, options):
comments.append(TRANSLATION_FLAG_COMMENT)
yield (message_lineno, funcname, messages, comments)
def extract_mxml(fileobj, keywords, comment_tags, options):
attrs = set(options.get('attrs', [])).union(
set([u'label', u'text', u'title', u'headerText', u'prompt']))
encoding = options.get('encoding', 'utf-8')
for elem, pos in MXMLParser(fileobj).parse():
if elem.tag == 'mx:Script':
for lineno, funcname, message, comments in \
extract_javascript(StringIO(elem.text), keywords,
comment_tags, options):
yield pos[0]+lineno, funcname, message, comments
else:
attrib = None
for attr in attrs:
if elem.get(attr):
attrib = attr
if attrib:
if elem.attrib[attrib].startswith('{') and \
elem.attrib[attrib].endswith('}'):
contents = elem.attrib[attrib][1:-1].encode(encoding)
for _, funcname, message, comments in extract_actionscript(
StringIO(contents), keywords, comment_tags, options):
yield pos[0], funcname, message, comments
def test_misplaced_comments():
buf = BytesIO(b"""\
/* NOTE: this won't show up */
foo()
/* NOTE: this will */
msg = _('Something')
// NOTE: this will show up
// too.
msg = _('Something else')
// NOTE: but this won't
bar()
_('no comment here')
""")
messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
assert messages[0][2] == u'Something'
assert messages[0][3] == [u'NOTE: this will']
assert messages[1][2] == u'Something else'
assert messages[1][3] == [u'NOTE: this will show up', 'too.']
assert messages[2][2] == u'no comment here'
assert messages[2][3] == []
def test_misplaced_comments(self):
buf = StringIO("""\
/* NOTE: this won't show up */
foo()
/* NOTE: this will */
msg = _('Something')
// NOTE: this will show up
// too.
msg = _('Something else')
// NOTE: but this won't
bar()
_('no comment here')
""")
messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
self.assertEqual(u'Something', messages[0][2])
self.assertEqual([u'NOTE: this will'], messages[0][3])
self.assertEqual(u'Something else', messages[1][2])
self.assertEqual([u'NOTE: this will show up', 'too.'], messages[1][3])
self.assertEqual(u'no comment here', messages[2][2])
self.assertEqual([], messages[2][3])
def extract_extrajs(fileobj, keywords, comment_tags, options):
"""Extract template literal placeholders and filters from Javascript files.
:param fileobj: the file-like the messages should be extracted from
:param keywords: a list of keywords (i.e. function names) that should be recognize as translation functions
:param comment_tags: a list of translator tags to search for and include in the results
:param options: a dictionary of additional options (optional)
:return: an iterator over ``(lineno, funcname, message, comments)``
:rtype: ``iterator``
"""
encoding = options.get('encoding', 'utf-8')
c = fileobj.read().decode(encoding=encoding)
filtpat = re.compile('t\._f\("gettext"\)', re.UNICODE)
c = filtpat.sub('gettext', c)
comppat = re.compile(r'(\$\{gettext\((.*?)\)\})', re.UNICODE)
c = comppat.sub(r'`+gettext(\g<2>)+`', c, re.UNICODE)
for i in extract_javascript(BytesIO(c.encode(encoding=encoding)),
DEFAULT_KEYWORDS.keys(), comment_tags,
options):
if i:
yield (i[0], i[1], i[2], i[3])
def test_misplaced_comments():
buf = BytesIO(b"""\
/* NOTE: this won't show up */
foo()
/* NOTE: this will */
msg = _('Something')
// NOTE: this will show up
// too.
msg = _('Something else')
// NOTE: but this won't
bar()
_('no comment here')
""")
messages = list(extract.extract_javascript(buf, ('_', ), ['NOTE:'], {}))
assert messages[0][2] == u'Something'
assert messages[0][3] == [u'NOTE: this will']
assert messages[1][2] == u'Something else'
assert messages[1][3] == [u'NOTE: this will show up', 'too.']
assert messages[2][2] == u'no comment here'
assert messages[2][3] == []
