def extract_vue(fileobj, keywords, comment_tags, options): """Extract messages from Vue template files. :param fileobj: the file-like the messages should be extracted from :param keywords: a list of keywords (i.e. function names) that should be recognize as translation functions :param comment_tags: a list of translator tags to search for and include in the results :param options: a dictionary of additional options (optional) :return: an iterator over ``(lineno, funcname, message, comments)`` :rtype: ``iterator`` """ encoding = options.get('encoding', 'utf-8') contents = fileobj.read().decode(encoding=encoding) lexer = Lexer(contents, None) for t in lexer.tokenize(): # type: Token if t.token_type in TOKENS: for i in extract_javascript( BytesIO(t.contents.encode(encoding=encoding)), keywords, comment_tags, options): if i: yield (t.lineno, i[1], i[2], i[3]) # Inline JS part matchjs = re.search(r'<script>(.+)</script>', contents, re.DOTALL) if not matchjs: return jscontent = matchjs.group(1) skipped_line_count = contents[:matchjs.start(1)].count('\n') for i in extract_javascript(BytesIO(jscontent.encode(encoding=encoding)), keywords, comment_tags, options): yield (skipped_line_count + i[0], i[1], i[2], i[3])
def test_jsx_extraction(jsx_enabled): buf = BytesIO(JSX_SOURCE) messages = [m[2] for m in extract.extract_javascript(buf, ('_', 'gettext'), [], {"jsx": jsx_enabled})] if jsx_enabled: assert messages == EXPECTED_JSX_MESSAGES else: assert messages != EXPECTED_JSX_MESSAGES
def test_misplaced_comments(self): buf = BytesIO( b"""\ /* NOTE: this won't show up */ foo() /* NOTE: this will */ msg = _('Something') // NOTE: this will show up // too. msg = _('Something else') // NOTE: but this won't bar() _('no comment here') """ ) messages = list(extract.extract_javascript(buf, ("_",), ["NOTE:"], {})) self.assertEqual(u"Something", messages[0][2]) self.assertEqual([u"NOTE: this will"], messages[0][3]) self.assertEqual(u"Something else", messages[1][2]) self.assertEqual([u"NOTE: this will show up", "too."], messages[1][3]) self.assertEqual(u"no comment here", messages[2][2]) self.assertEqual([], messages[2][3])
def test_misplaced_comments(self): buf = StringIO("""\ /* NOTE: this won't show up */ foo() /* NOTE: this will */ msg = _('Something') // NOTE: this will show up // too. msg = _('Something else') // NOTE: but this won't bar() _('no comment here') """) messages = list(extract.extract_javascript(buf, ('_', ), ['NOTE:'], {})) self.assertEqual(u'Something', messages[0][2]) self.assertEqual([u'NOTE: this will'], messages[0][3]) self.assertEqual(u'Something else', messages[1][2]) self.assertEqual([u'NOTE: this will show up', 'too.'], messages[1][3]) self.assertEqual(u'no comment here', messages[2][2]) self.assertEqual([], messages[2][3])
def test_ignore_function_definitions(): buf = BytesIO(b"""\ function gettext(value) { return translations[language][value] || value; }""") messages = list(extract.extract_javascript(buf, ('gettext',), [], {})) assert not messages
def test_message_with_line_comment(self): buf = BytesIO(u"""\ // NOTE: hello msg = _('Bonjour à tous') """.encode('utf-8')) messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {})) self.assertEqual(u'Bonjour à tous', messages[0][2]) self.assertEqual([u'NOTE: hello'], messages[0][3])
def test_message_with_line_comment(self): buf = StringIO("""\ // NOTE: hello msg = _('Bonjour à tous') """) messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {})) self.assertEqual('Bonjour \xe0 tous', messages[0][2]) self.assertEqual(['NOTE: hello'], messages[0][3])
def test_ignore_function_definitions(self): buf = StringIO("""\ function gettext(value) { return translations[language][value] || value; }""") messages = list(extract.extract_javascript(buf, ('gettext',), [], {})) self.assertEqual(messages, [])
def test_ignore_function_definitions(self): buf = StringIO("""\ function gettext(value) { return translations[language][value] || value; }""") messages = list(extract.extract_javascript(buf, ('gettext', ), [], {})) self.assertEqual(messages, [])
def test_message_with_line_comment(): buf = BytesIO(u"""\ // NOTE: hello msg = _('Bonjour à tous') """.encode('utf-8')) messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {})) assert messages[0][2] == u'Bonjour à tous' assert messages[0][3] == [u'NOTE: hello']
def test_message_with_line_comment(): buf = BytesIO(u"""\ // NOTE: hello msg = _('Bonjour à tous') """.encode('utf-8')) messages = list(extract.extract_javascript(buf, ('_', ), ['NOTE:'], {})) assert messages[0][2] == u'Bonjour à tous' assert messages[0][3] == [u'NOTE: hello']
def extract_javascript_msgids(source): """Return message ids of translateable strings in JS source.""" extracted = extract_javascript( fileobj=StringIO(source), keywords={"_": None, "P_": (1, 2), "N_": None}, comment_tags={}, options={} ) return [msg_id for line, func, msg_id, comments in extracted]
def test_ignore_function_definitions(): buf = BytesIO(b"""\ function gettext(value) { return translations[language][value] || value; }""") messages = list(extract.extract_javascript(buf, ('gettext', ), [], {})) assert not messages
def test_message_with_multiple_line_comments(self): buf = StringIO("""\ // NOTE: hello // goodbye msg = _('Bonjour à tous') """) messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {})) self.assertEqual(u'Bonjour à tous', messages[0][2]) self.assertEqual([u'NOTE: hello goodbye'], messages[0][3])
def test_message_with_line_comment(self): buf = StringIO("""\ // NOTE: hello msg = _('Bonjour à tous') """) messages = list(extract.extract_javascript(buf, ('_', ), ['NOTE:'], {})) self.assertEqual(u'Bonjour à tous', messages[0][2]) self.assertEqual([u'NOTE: hello'], messages[0][3])
def test_jsx_extraction(jsx_enabled): buf = BytesIO(JSX_SOURCE) messages = [ m[2] for m in extract.extract_javascript(buf, ( '_', 'gettext'), [], {"jsx": jsx_enabled}) ] if jsx_enabled: assert messages == EXPECTED_JSX_MESSAGES else: assert messages != EXPECTED_JSX_MESSAGES
def test_message_with_multiline_comment(self): buf = StringIO("""\ /* NOTE: hello and bonjour and servus */ msg = _('Bonjour à tous') """) messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {})) self.assertEqual(u'Bonjour à tous', messages[0][2]) self.assertEqual([u'NOTE: hello', 'and bonjour', ' and servus'], messages[0][3])
def test_message_with_multiline_comment(self): buf = BytesIO(u"""\ /* NOTE: hello and bonjour and servus */ msg = _('Bonjour à tous') """.encode('utf-8')) messages = list(extract.extract_javascript(buf, ('_', ), ['NOTE:'], {})) self.assertEqual(u'Bonjour à tous', messages[0][2]) self.assertEqual([u'NOTE: hello', 'and bonjour', ' and servus'], messages[0][3])
def test_message_with_line_comment(self): buf = BytesIO( u"""\ // NOTE: hello msg = _('Bonjour à tous') """.encode( "utf-8" ) ) messages = list(extract.extract_javascript(buf, ("_",), ["NOTE:"], {})) self.assertEqual(u"Bonjour à tous", messages[0][2]) self.assertEqual([u"NOTE: hello"], messages[0][3])
def test_message_with_multiline_comment(self): buf = StringIO("""\ /* NOTE: hello and bonjour and servus */ msg = _('Bonjour à tous') """) messages = list(extract.extract_javascript(buf, ('_', ), ['NOTE:'], {})) self.assertEqual(u('Bonjour \xe0 tous'), messages[0][2]) self.assertEqual([u('NOTE: hello'), 'and bonjour', ' and servus'], messages[0][3])
def extract_javascript_script(fileobj, keywords, comment_tags, options): """Extract messages from Javascript embedding in <script> tags. Select <script type="javascript/text"> tags and delegate to `extract_javascript`. """ from genshi.core import Stream from genshi.input import XMLParser out = StringIO() stream = Stream(XMLParser(fileobj)) stream.select('//script[@type="text/javascript"]').render(out=out) out.seek(0) return extract_javascript(out, keywords, comment_tags, options)
def extract_javascript_script(fileobj, keywords, comment_tags, options): """Extract messages from Javascript embedded in <script> tags. Select <script type="javascript/text"> tags and delegate to `extract_javascript`. """ if not fileobj.name: return [] out = io.StringIO() extractor = ScriptExtractor(out) extractor.feed(unicode(fileobj.read(), 'utf-8')) extractor.close() out.seek(0) return extract_javascript(out, keywords, comment_tags, options)
def extract_javascript_msgids(source): """Return message ids of translateable strings in JS source.""" extracted = extract_javascript( fileobj=StringIO(source), keywords={ "_": None, "P_": (1, 2), "N_": None, }, comment_tags={}, options={}, ) return [msg_id for line, func, msg_id, comments in extracted]
def extract_mxml(fileobj, keywords, comment_tags, options): for elem, pos in MXMLParser(fileobj).parse(): if elem.tag == 'mx:Script': for lineno, funcname, message, comments in \ extract_javascript(StringIO(elem.text), keywords, comment_tags, options): yield pos[0]+lineno, funcname, message, comments else: attrib = None for attr in options.get('attrs', []): if elem.get(attr): attrib = attr if attrib: if elem.attrib[attrib].startswith('{') and \ elem.attrib[attrib].endswith('}'): for _, funcname, message, comments in extract_actionscript( StringIO(elem.attrib[attrib][1:-1]), keywords, comment_tags, options): yield pos[0], funcname, message, comments
def extract_vue(fileobj, keywords, comment_tags, options): """Extract messages from Vue template files. :param fileobj: the file-like the messages should be extracted from :param keywords: a list of keywords (i.e. function names) that should be recognize as translation functions :param comment_tags: a list of translator tags to search for and include in the results :param options: a dictionary of additional options (optional) :return: an iterator over ``(lineno, funcname, message, comments)`` :rtype: ``iterator`` """ encoding = options.get('encoding', 'utf-8') contents = fileobj.read().decode(encoding=encoding) lexer = Lexer(contents, None) for t in lexer.tokenize(): # type: Token if t.token_type in TOKENS: for i in extract_javascript( BytesIO(t.contents.encode(encoding=encoding)), keywords, comment_tags, options): if i: yield (t.lineno, i[1], i[2], i[3])
def extract_javascript(fileobj, keywords, comment_tags, options): """Extract messages from Javascript source files. This extractor delegates to babel's buit-in javascript extractor, but adds a special comment used as a flag to identify web translations. :param fileobj: the file-like object the messages should be extracted from :param keywords: a list of keywords (i.e. function names) that should be recognized as translation functions :param comment_tags: a list of translator tags to search for and include in the results :param options: a dictionary of additional options (optional) :return: an iterator over ``(lineno, funcname, message, comments)`` tuples :rtype: ``iterator`` """ for (message_lineno, funcname, messages, comments) in \ extract.extract_javascript(fileobj, keywords, comment_tags, options): comments.append(TRANSLATION_FLAG_COMMENT) yield (message_lineno, funcname, messages, comments)
def extract_mxml(fileobj, keywords, comment_tags, options): attrs = set(options.get('attrs', [])).union( set([u'label', u'text', u'title', u'headerText', u'prompt'])) encoding = options.get('encoding', 'utf-8') for elem, pos in MXMLParser(fileobj).parse(): if elem.tag == 'mx:Script': for lineno, funcname, message, comments in \ extract_javascript(StringIO(elem.text), keywords, comment_tags, options): yield pos[0]+lineno, funcname, message, comments else: attrib = None for attr in attrs: if elem.get(attr): attrib = attr if attrib: if elem.attrib[attrib].startswith('{') and \ elem.attrib[attrib].endswith('}'): contents = elem.attrib[attrib][1:-1].encode(encoding) for _, funcname, message, comments in extract_actionscript( StringIO(contents), keywords, comment_tags, options): yield pos[0], funcname, message, comments
def test_misplaced_comments(): buf = BytesIO(b"""\ /* NOTE: this won't show up */ foo() /* NOTE: this will */ msg = _('Something') // NOTE: this will show up // too. msg = _('Something else') // NOTE: but this won't bar() _('no comment here') """) messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {})) assert messages[0][2] == u'Something' assert messages[0][3] == [u'NOTE: this will'] assert messages[1][2] == u'Something else' assert messages[1][3] == [u'NOTE: this will show up', 'too.'] assert messages[2][2] == u'no comment here' assert messages[2][3] == []
def test_misplaced_comments(self): buf = StringIO("""\ /* NOTE: this won't show up */ foo() /* NOTE: this will */ msg = _('Something') // NOTE: this will show up // too. msg = _('Something else') // NOTE: but this won't bar() _('no comment here') """) messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {})) self.assertEqual(u'Something', messages[0][2]) self.assertEqual([u'NOTE: this will'], messages[0][3]) self.assertEqual(u'Something else', messages[1][2]) self.assertEqual([u'NOTE: this will show up', 'too.'], messages[1][3]) self.assertEqual(u'no comment here', messages[2][2]) self.assertEqual([], messages[2][3])
def extract_extrajs(fileobj, keywords, comment_tags, options): """Extract template literal placeholders and filters from Javascript files. :param fileobj: the file-like the messages should be extracted from :param keywords: a list of keywords (i.e. function names) that should be recognize as translation functions :param comment_tags: a list of translator tags to search for and include in the results :param options: a dictionary of additional options (optional) :return: an iterator over ``(lineno, funcname, message, comments)`` :rtype: ``iterator`` """ encoding = options.get('encoding', 'utf-8') c = fileobj.read().decode(encoding=encoding) filtpat = re.compile('t\._f\("gettext"\)', re.UNICODE) c = filtpat.sub('gettext', c) comppat = re.compile(r'(\$\{gettext\((.*?)\)\})', re.UNICODE) c = comppat.sub(r'`+gettext(\g<2>)+`', c, re.UNICODE) for i in extract_javascript(BytesIO(c.encode(encoding=encoding)), DEFAULT_KEYWORDS.keys(), comment_tags, options): if i: yield (i[0], i[1], i[2], i[3])
def test_misplaced_comments(): buf = BytesIO(b"""\ /* NOTE: this won't show up */ foo() /* NOTE: this will */ msg = _('Something') // NOTE: this will show up // too. msg = _('Something else') // NOTE: but this won't bar() _('no comment here') """) messages = list(extract.extract_javascript(buf, ('_', ), ['NOTE:'], {})) assert messages[0][2] == u'Something' assert messages[0][3] == [u'NOTE: this will'] assert messages[1][2] == u'Something else' assert messages[1][3] == [u'NOTE: this will show up', 'too.'] assert messages[2][2] == u'no comment here' assert messages[2][3] == []