def _extract_attrs(self, event, gettext_functions, search_text): for name, value in event[1][1]: if search_text and isinstance(value, basestring): if name in self.include_attrs: text = value.strip() if text: yield event[2][1], None, text, [] else: for message in self.extract(_ensure(value), gettext_functions, search_text=False): yield message
def _eval(self, stream, ctxt, **vars): """Internal stream filter that evaluates any expressions in `START` and `TEXT` events. """ filters = (self._flatten, self._eval) number_conv = self._number_conv for kind, data, pos in stream: if kind is START and data[1]: # Attributes may still contain expressions in start tags at # this point, so do some evaluation tag, attrs = data new_attrs = [] for name, substream in attrs: if isinstance(substream, basestring): value = substream else: values = [] for subkind, subdata, subpos in self._eval(substream, ctxt, **vars): if subkind is TEXT: values.append(subdata) value = [x for x in values if x is not None] if not value: continue new_attrs.append((name, u''.join(value))) yield kind, (tag, Attrs(new_attrs)), pos elif kind is EXPR: result = _eval_expr(data, ctxt, **vars) if result is not None: # First check for a string, otherwise the iterable test # below succeeds, and the string will be chopped up into # individual characters if isinstance(result, basestring): yield TEXT, result, pos elif isinstance(result, (int, float, long)): yield TEXT, number_conv(result), pos elif hasattr(result, '__iter__'): substream = _ensure(result) for filter_ in filters: substream = filter_(substream, ctxt, **vars) for event in substream: yield event else: yield TEXT, unicode(result), pos else: yield kind, data, pos
def output (self, template, **kwargs): """ Output the page. """ # Send HTTP content-type header. cherrypy.response.headers['Content-Type'] = self.CONTENT_TYPE template = Formatters.formatters[self.FORMATTER].templates[template] ctxt = genshi.template.Context (cherrypy = cherrypy, bs = BaseSearcher, **kwargs) stream = template.stream for filter_ in template.filters: stream = filter_ (iter (stream), ctxt) serializer = self.get_serializer () return genshi.output.encode (serializer ( _ensure (genshi.Stream (stream))), encoding = 'utf-8')
def render(self, page, os): """ Render and send to browser. """ self.send_headers() template = self.templates[page] ctxt = genshi.template.Context(cherrypy=cherrypy, os=os, bs=BaseSearcher) stream = template.stream for filter_ in template.filters: stream = filter_(iter(stream), ctxt) # there's no easy way in genshi to pass collapse_lines to this filter stream = WHITESPACE_FILTER(stream, collapse_lines=COLLAPSE_LINES) return genshi.output.encode(self.get_serializer()(_ensure( genshi.Stream(stream))), encoding='utf-8')
def __call__(self, stream, ctxt=None, translate_text=True, translate_attrs=True): """Translate any localizable strings in the given stream. This function shouldn't be called directly. Instead, an instance of the `Translator` class should be registered as a filter with the `Template` or the `TemplateLoader`, or applied as a regular stream filter. If used as a template filter, it should be inserted in front of all the default filters. :param stream: the markup event stream :param ctxt: the template context (not used) :param translate_text: whether text nodes should be translated (used internally) :param translate_attrs: whether attribute values should be translated (used internally) :return: the localized stream """ ignore_tags = self.ignore_tags include_attrs = self.include_attrs skip = 0 xml_lang = XML_NAMESPACE['lang'] if not self.extract_text: translate_text = False translate_attrs = False if type(self.translate) is FunctionType: gettext = self.translate if ctxt: ctxt['_i18n.gettext'] = gettext else: if IS_PYTHON2: gettext = self.translate.ugettext ngettext = self.translate.ungettext else: gettext = self.translate.gettext ngettext = self.translate.ngettext try: if IS_PYTHON2: dgettext = self.translate.dugettext dngettext = self.translate.dungettext else: dgettext = self.translate.dgettext dngettext = self.translate.dngettext except AttributeError: dgettext = lambda _, y: gettext(y) dngettext = lambda _, s, p, n: ngettext(s, p, n) if ctxt: ctxt['_i18n.gettext'] = gettext ctxt['_i18n.ngettext'] = ngettext ctxt['_i18n.dgettext'] = dgettext ctxt['_i18n.dngettext'] = dngettext if ctxt and ctxt.get('_i18n.domain'): # TODO: This can cause infinite recursion if dgettext is defined # via the AttributeError case above! gettext = lambda msg: dgettext(ctxt.get('_i18n.domain'), msg) for kind, data, pos in stream: # skip chunks that should not be localized if skip: if kind is START: skip += 1 elif kind is END: skip -= 1 yield kind, data, pos continue # handle different events that can be localized if kind is START: tag, attrs = data if tag in self.ignore_tags or \ isinstance(attrs.get(xml_lang), basestring): skip += 1 yield kind, data, pos continue new_attrs = [] changed = False for name, value in attrs: newval = value if isinstance(value, basestring): if translate_attrs and name in include_attrs: newval = gettext(value) else: newval = list( self(_ensure(value), ctxt, translate_text=False) ) if newval != value: value = newval changed = True new_attrs.append((name, value)) if changed: attrs = Attrs(new_attrs) yield kind, (tag, attrs), pos elif translate_text and kind is TEXT: text = data.strip() if text: data = data.replace(text, unicode(gettext(text))) yield kind, data, pos elif kind is SUB: directives, substream = data current_domain = None for idx, directive in enumerate(directives): # Organize directives to make everything work # FIXME: There's got to be a better way to do this! if isinstance(directive, DomainDirective): # Grab current domain and update context current_domain = directive.domain ctxt.push({'_i18n.domain': current_domain}) # Put domain directive as the first one in order to # update context before any other directives evaluation directives.insert(0, directives.pop(idx)) # If this is an i18n directive, no need to translate text # nodes here is_i18n_directive = any([ isinstance(d, ExtractableI18NDirective) for d in directives ]) substream = list(self(substream, ctxt, translate_text=not is_i18n_directive, translate_attrs=translate_attrs)) yield kind, (directives, substream), pos if current_domain: ctxt.pop() else: yield kind, data, pos
def extract(self, stream, gettext_functions=GETTEXT_FUNCTIONS, search_text=True): """Extract localizable strings from the given template stream. For every string found, this function yields a ``(lineno, function, message)`` tuple, where: * ``lineno`` is the number of the line on which the string was found, * ``function`` is the name of the ``gettext`` function used (if the string was extracted from embedded Python code), and * ``message`` is the string itself (a ``unicode`` object, or a tuple of ``unicode`` objects for functions with multiple string arguments). >>> from genshi.template import MarkupTemplate >>> >>> tmpl = MarkupTemplate('''<html xmlns:py="http://genshi.edgewall.org/"> ... <head> ... <title>Example</title> ... </head> ... <body> ... <h1>Example</h1> ... <p>${_("Hello, %(name)s") % dict(name=username)}</p> ... <p>${ngettext("You have %d item", "You have %d items", num)}</p> ... </body> ... </html>''', filename='example.html') >>> >>> for lineno, funcname, message in Translator().extract(tmpl.stream): ... print "%d, %r, %r" % (lineno, funcname, message) 3, None, u'Example' 6, None, u'Example' 7, '_', u'Hello, %(name)s' 8, 'ngettext', (u'You have %d item', u'You have %d items') :param stream: the event stream to extract strings from; can be a regular stream or a template stream :param gettext_functions: a sequence of function names that should be treated as gettext-style localization functions :param search_text: whether the content of text nodes should be extracted (used internally) :note: Changed in 0.4.1: For a function with multiple string arguments (such as ``ngettext``), a single item with a tuple of strings is yielded, instead an item for each string argument. """ tagname = None if not self.extract_text: search_text = False skip = 0 xml_lang = XML_NAMESPACE['lang'] for kind, data, pos in stream: if skip: if kind is START: skip += 1 if kind is END: skip -= 1 if kind is START and not skip: tag, attrs = data if tag in self.ignore_tags or \ isinstance(attrs.get(xml_lang), basestring): skip += 1 continue for name, value in attrs: if search_text and isinstance(value, basestring): if name in self.include_attrs: text = value.strip() if text: yield pos[1], None, text else: for lineno, funcname, text in self.extract( _ensure(value), gettext_functions, search_text=False): yield lineno, funcname, text elif not skip and search_text and kind is TEXT: text = data.strip() if text and filter(None, [ch.isalpha() for ch in text]): yield pos[1], None, text elif kind is EXPR or kind is EXEC: consts = dict([(n, chr(i) + '\x00') for i, n in enumerate(data.code.co_consts)]) gettext_locs = [ consts[n] for n in gettext_functions if n in consts ] ops = [ _LOAD_CONST, '(', '|'.join(gettext_locs), ')', _CALL_FUNCTION, '.\x00', '((?:', _BINARY_ADD, '|', _LOAD_CONST, '.\x00)+)' ] for loc, opcodes in re.findall(''.join(ops), data.code.co_code): funcname = data.code.co_consts[ord(loc[0])] strings = [] opcodes = iter(opcodes) for opcode in opcodes: if opcode == _BINARY_ADD: arg = strings.pop() strings[-1] += arg else: arg = data.code.co_consts[ord(opcodes.next())] opcodes.next() # skip second byte if not isinstance(arg, basestring): break strings.append(unicode(arg)) if len(strings) == 1: strings = strings[0] else: strings = tuple(strings) yield pos[1], funcname, strings elif kind is SUB: subkind, substream = data messages = self.extract(substream, gettext_functions, search_text=search_text and not skip) for lineno, funcname, text in messages: yield lineno, funcname, text
def __call__(self, stream, ctxt=None, search_text=True): """Translate any localizable strings in the given stream. This function shouldn't be called directly. Instead, an instance of the `Translator` class should be registered as a filter with the `Template` or the `TemplateLoader`, or applied as a regular stream filter. If used as a template filter, it should be inserted in front of all the default filters. :param stream: the markup event stream :param ctxt: the template context (not used) :param search_text: whether text nodes should be translated (used internally) :return: the localized stream """ ignore_tags = self.ignore_tags include_attrs = self.include_attrs translate = self.translate if not self.extract_text: search_text = False skip = 0 xml_lang = XML_NAMESPACE['lang'] for kind, data, pos in stream: # skip chunks that should not be localized if skip: if kind is START: skip += 1 elif kind is END: skip -= 1 yield kind, data, pos continue # handle different events that can be localized if kind is START: tag, attrs = data if tag in self.ignore_tags or \ isinstance(attrs.get(xml_lang), basestring): skip += 1 yield kind, data, pos continue new_attrs = [] changed = False for name, value in attrs: newval = value if search_text and isinstance(value, basestring): if name in include_attrs: newval = self.translate(value) else: newval = list( self(_ensure(value), ctxt, search_text=False)) if newval != value: value = newval changed = True new_attrs.append((name, value)) if changed: attrs = new_attrs yield kind, (tag, attrs), pos elif search_text and kind is TEXT: text = data.strip() if text: data = data.replace(text, translate(text)) yield kind, data, pos elif kind is SUB: subkind, substream = data new_substream = list(self(substream, ctxt)) yield kind, (subkind, new_substream), pos else: yield kind, data, pos
def extract(self, stream, gettext_functions=GETTEXT_FUNCTIONS, search_text=True, msgbuf=None, comment_stack=None): """Extract localizable strings from the given template stream. For every string found, this function yields a ``(lineno, function, message, comments)`` tuple, where: * ``lineno`` is the number of the line on which the string was found, * ``function`` is the name of the ``gettext`` function used (if the string was extracted from embedded Python code), and * ``message`` is the string itself (a ``unicode`` object, or a tuple of ``unicode`` objects for functions with multiple string arguments). * ``comments`` is a list of comments related to the message, extracted from ``i18n:comment`` attributes found in the markup >>> tmpl = MarkupTemplate('''<html xmlns:py="http://genshi.edgewall.org/"> ... <head> ... <title>Example</title> ... </head> ... <body> ... <h1>Example</h1> ... <p>${_("Hello, %(name)s") % dict(name=username)}</p> ... <p>${ngettext("You have %d item", "You have %d items", num)}</p> ... </body> ... </html>''', filename='example.html') >>> for line, func, msg, comments in Translator().extract(tmpl.stream): ... print('%d, %r, %r' % (line, func, msg)) 3, None, u'Example' 6, None, u'Example' 7, '_', u'Hello, %(name)s' 8, 'ngettext', (u'You have %d item', u'You have %d items', None) :param stream: the event stream to extract strings from; can be a regular stream or a template stream :param gettext_functions: a sequence of function names that should be treated as gettext-style localization functions :param search_text: whether the content of text nodes should be extracted (used internally) :note: Changed in 0.4.1: For a function with multiple string arguments (such as ``ngettext``), a single item with a tuple of strings is yielded, instead an item for each string argument. :note: Changed in 0.6: The returned tuples now include a fourth element, which is a list of comments for the translator. """ if not self.extract_text: search_text = False if comment_stack is None: comment_stack = [] skip = 0 # Un-comment bellow to extract messages without adding directives xml_lang = XML_NAMESPACE['lang'] for kind, data, pos in stream: if skip: if kind is START: skip += 1 if kind is END: skip -= 1 if kind is START and not skip: tag, attrs = data if tag in self.ignore_tags or \ isinstance(attrs.get(xml_lang), basestring): skip += 1 continue for name, value in attrs: if search_text and isinstance(value, basestring): if name in self.include_attrs: text = value.strip() if text: # XXX: Do we need to grab i18n:comment from comment_stack ??? yield pos[1], None, text, [] else: for lineno, funcname, text, comments in self.extract( _ensure(value), gettext_functions, search_text=False): yield lineno, funcname, text, comments if msgbuf: msgbuf.append(kind, data, pos) elif not skip and search_text and kind is TEXT: if not msgbuf: text = data.strip() if text and [ch for ch in text if ch.isalpha()]: yield pos[1], None, text, comment_stack[-1:] else: msgbuf.append(kind, data, pos) elif not skip and msgbuf and kind is END: msgbuf.append(kind, data, pos) if not msgbuf.depth: yield msgbuf.lineno, None, msgbuf.format(), [ c for c in msgbuf.comment if c ] msgbuf = None elif kind is EXPR or kind is EXEC: if msgbuf: msgbuf.append(kind, data, pos) for funcname, strings in extract_from_code(data, gettext_functions): # XXX: Do we need to grab i18n:comment from comment_stack ??? yield pos[1], funcname, strings, [] elif kind is SUB: directives, substream = data in_comment = False for idx, directive in enumerate(directives): # Do a first loop to see if there's a comment directive # If there is update context and pop it from directives if isinstance(directive, CommentDirective): in_comment = True comment_stack.append(directive.comment) if len(directives) == 1: # in case we're in the presence of something like: # <p i18n:comment="foo">Foo</p> messages = self.extract( substream, gettext_functions, search_text=search_text and not skip, msgbuf=msgbuf, comment_stack=comment_stack) for lineno, funcname, text, comments in messages: yield lineno, funcname, text, comments directives.pop(idx) elif not isinstance(directive, I18NDirective): # Remove all other non i18n directives from the process directives.pop(idx) if not directives and not in_comment: # Extract content if there's no directives because # strip was pop'ed and not because comment was pop'ed. # Extraction in this case has been taken care of. messages = self.extract( substream, gettext_functions, search_text=search_text and not skip, msgbuf=msgbuf) for lineno, funcname, text, comments in messages: yield lineno, funcname, text, comments for directive in directives: if isinstance(directive, ExtractableI18NDirective): messages = directive.extract(substream, comment_stack) for funcname, text, comments in messages: yield pos[1], funcname, text, comments else: messages = self.extract( substream, gettext_functions, search_text=search_text and not skip, msgbuf=msgbuf) for lineno, funcname, text, comments in messages: yield lineno, funcname, text, comments if in_comment: comment_stack.pop()
def _inject(self): content = self.content if hasattr(content, '__call__'): content = content() for event in _ensure(content): yield None, event
def __call__(self, stream): """Apply the transform filter to the marked stream. :param stream: The marked event stream to filter """ found_first_th = False if self.data['xhr']: in_dirlist = True # XHR rows are only the interesting table else: in_dirlist = False in_repoindex = in_name_td = False idx = 0 rows_seen = 0 for kind, data, pos in stream: if kind == START: if all((data[0] == QName("http://www.w3.org/1999/xhtml}table"), data[1].get('id') == 'dirlist', self.data['dir'])): in_dirlist = True in_repoindex = data[1].get('id') == 'repoindex' if all((in_dirlist, not found_first_th, data[0] == QName("http://www.w3.org/1999/xhtml}th"))): for event in _ensure(tag.th(Markup(' '))): yield event found_first_th = True yield kind, data, pos continue if in_dirlist and data[0] == QName("http://www.w3.org/1999/xhtml}td"): rows_seen = rows_seen + 1 if rows_seen == 1 and ('up' in self.data['chrome']['links'] or \ (self.data['dir'] and not self.data['dir']['entries'])): data = data[0], data[1] - 'colspan' | [(QName('colspan'), '7')] yield kind, data, pos continue # don't mess with the "parent link" and "No files found" row if data[1].get('class') == 'name': # would be nice to get this static for any particular # item. We can't use a simple offset count due # to the XHR requests uid = uuid.uuid4() for event in _ensure(tag.td(tag.input(type='checkbox', id="cb-%s" % uid, class_='fileselect'))): yield event in_name_td = True elif in_dirlist and kind == END and data == QName("http://www.w3.org/1999/xhtml}table"): # we're leaving the current table; reset markers in_dirlist = False rows_seen = 0 found_first_th = False idx = 0 elif in_name_td and kind == END: if data == QName("http://www.w3.org/1999/xhtml}td"): in_name_td = False elif data == QName("http://www.w3.org/1999/xhtml}a"): yield kind, data, pos if idx == 0 and in_repoindex: # Don't yield a context menu for the repos since they don't have a dir entry continue entry = self.data['dir']['entries'][idx] menu = tag.div(tag.a(class_="ctx-expander fa fa-angle-down"), tag.div( tag.ul(tag.li(tag.span("%s (version %d)" % (entry.name, entry.rev), class_="plain-text")), class_="styled-dropdown"), class_="bottom-fix" ), id="ctx-%s" % uid, class_="inline-block margin-left-small dropdown-toggle") for provider in sorted(self.context_menu_providers, key=lambda x: x.get_order(self.req)): content = provider.get_content(self.req, entry, self.data) if content: menu.children[1].children[0].append(tag.li(content)) if len(menu.children[1].children[0].children) > 1: for event in _ensure(menu): yield event idx = idx + 1 continue yield kind, data, pos
def extract(self, stream, gettext_functions=GETTEXT_FUNCTIONS, search_text=True, msgbuf=None): """Extract localizable strings from the given template stream. For every string found, this function yields a ``(lineno, function, message, comments)`` tuple, where: * ``lineno`` is the number of the line on which the string was found, * ``function`` is the name of the ``gettext`` function used (if the string was extracted from embedded Python code), and * ``message`` is the string itself (a ``unicode`` object, or a tuple of ``unicode`` objects for functions with multiple string arguments). * ``comments`` is a list of comments related to the message, extracted from ``i18n:comment`` attributes found in the markup >>> from genshi.template import MarkupTemplate >>> >>> tmpl = MarkupTemplate('''<html xmlns:py="http://genshi.edgewall.org/"> ... <head> ... <title>Example</title> ... </head> ... <body> ... <h1>Example</h1> ... <p>${_("Hello, %(name)s") % dict(name=username)}</p> ... <p>${ngettext("You have %d item", "You have %d items", num)}</p> ... </body> ... </html>''', filename='example.html') >>> >>> for line, func, msg, comments in Translator().extract(tmpl.stream): ... print "%d, %r, %r" % (line, func, msg) 3, None, u'Example' 6, None, u'Example' 7, '_', u'Hello, %(name)s' 8, 'ngettext', (u'You have %d item', u'You have %d items', None) :param stream: the event stream to extract strings from; can be a regular stream or a template stream :param gettext_functions: a sequence of function names that should be treated as gettext-style localization functions :param search_text: whether the content of text nodes should be extracted (used internally) :note: Changed in 0.4.1: For a function with multiple string arguments (such as ``ngettext``), a single item with a tuple of strings is yielded, instead an item for each string argument. :note: Changed in 0.6: The returned tuples now include a 4th element, which is a list of comments for the translator """ if not self.extract_text: search_text = False skip = 0 i18n_comment = I18N_NAMESPACE['comment'] i18n_msg = I18N_NAMESPACE['msg'] xml_lang = XML_NAMESPACE['lang'] for kind, data, pos in stream: if skip: if kind is START: skip += 1 if kind is END: skip -= 1 if kind is START and not skip: tag, attrs = data if tag in self.ignore_tags or \ isinstance(attrs.get(xml_lang), basestring): skip += 1 continue for name, value in attrs: if search_text and isinstance(value, basestring): if name in self.include_attrs: text = value.strip() if text: yield pos[1], None, text, [] else: for lineno, funcname, text, comments in self.extract( _ensure(value), gettext_functions, search_text=False): yield lineno, funcname, text, comments if msgbuf: msgbuf.append(kind, data, pos) else: msg_params = attrs.get(i18n_msg) if msg_params is not None: if type(msg_params) is list: # event tuple msg_params = msg_params[0][1] msgbuf = MessageBuffer(msg_params, attrs.get(i18n_comment), pos[1]) elif not skip and search_text and kind is TEXT: if not msgbuf: text = data.strip() if text and filter(None, [ch.isalpha() for ch in text]): yield pos[1], None, text, [] else: msgbuf.append(kind, data, pos) elif not skip and msgbuf and kind is END: msgbuf.append(kind, data, pos) if not msgbuf.depth: yield msgbuf.lineno, None, msgbuf.format(), \ filter(None, [msgbuf.comment]) msgbuf = None elif kind is EXPR or kind is EXEC: if msgbuf: msgbuf.append(kind, data, pos) for funcname, strings in extract_from_code( data, gettext_functions): yield pos[1], funcname, strings, [] elif kind is SUB: subkind, substream = data messages = self.extract(substream, gettext_functions, search_text=search_text and not skip, msgbuf=msgbuf) for lineno, funcname, text, comments in messages: yield lineno, funcname, text, comments
def __call__(self, stream, ctxt=None, search_text=True, msgbuf=None): """Translate any localizable strings in the given stream. This function shouldn't be called directly. Instead, an instance of the `Translator` class should be registered as a filter with the `Template` or the `TemplateLoader`, or applied as a regular stream filter. If used as a template filter, it should be inserted in front of all the default filters. :param stream: the markup event stream :param ctxt: the template context (not used) :param search_text: whether text nodes should be translated (used internally) :param msgbuf: a `MessageBuffer` object or `None` (used internally) :return: the localized stream """ ignore_tags = self.ignore_tags include_attrs = self.include_attrs translate = self.translate if not self.extract_text: search_text = False skip = 0 i18n_msg = I18N_NAMESPACE['msg'] ns_prefixes = [] xml_lang = XML_NAMESPACE['lang'] for kind, data, pos in stream: # skip chunks that should not be localized if skip: if kind is START: skip += 1 elif kind is END: skip -= 1 yield kind, data, pos continue # handle different events that can be localized if kind is START: tag, attrs = data if tag in self.ignore_tags or \ isinstance(attrs.get(xml_lang), basestring): skip += 1 yield kind, data, pos continue new_attrs = [] changed = False for name, value in attrs: newval = value if search_text and isinstance(value, basestring): if name in include_attrs: newval = self.translate(value) else: newval = list(self(_ensure(value), ctxt, search_text=False, msgbuf=msgbuf) ) if newval != value: value = newval changed = True new_attrs.append((name, value)) if changed: attrs = Attrs(new_attrs) if msgbuf: msgbuf.append(kind, data, pos) continue elif i18n_msg in attrs: params = attrs.get(i18n_msg) if params and type(params) is list: # event tuple params = params[0][1] msgbuf = MessageBuffer(params) attrs -= i18n_msg yield kind, (tag, attrs), pos elif search_text and kind is TEXT: if not msgbuf: text = data.strip() if text: data = data.replace(text, unicode(translate(text))) yield kind, data, pos else: msgbuf.append(kind, data, pos) elif msgbuf and kind is EXPR: msgbuf.append(kind, data, pos) elif not skip and msgbuf and kind is END: msgbuf.append(kind, data, pos) if not msgbuf.depth: for event in msgbuf.translate(translate(msgbuf.format())): yield event msgbuf = None yield kind, data, pos elif kind is SUB: subkind, substream = data new_substream = list(self(substream, ctxt, msgbuf=msgbuf)) yield kind, (subkind, new_substream), pos elif kind is START_NS and data[1] == I18N_NAMESPACE: ns_prefixes.append(data[0]) elif kind is END_NS and data in ns_prefixes: ns_prefixes.remove(data) else: yield kind, data, pos
def _inject(self): content = self.content if callable(content): content = content() for event in _ensure(content): yield None, event
def _flatten(self, stream, ctxt, **vars): number_conv = self._number_conv stack = [] push = stack.append pop = stack.pop stream = iter(stream) while 1: for kind, data, pos in stream: if kind is START and data[1]: # Attributes may still contain expressions in start tags at # this point, so do some evaluation tag, attrs = data new_attrs = [] for name, value in attrs: if type(value) is list: # this is an interpolated string values = [event[1] for event in self._flatten(value, ctxt, **vars) if event[0] is TEXT and event[1] is not None ] if not values: continue value = ''.join(values) new_attrs.append((name, value)) yield kind, (tag, Attrs(new_attrs)), pos elif kind is EXPR: result = _eval_expr(data, ctxt, vars) if result is not None: # First check for a string, otherwise the iterable test # below succeeds, and the string will be chopped up into # individual characters if isinstance(result, basestring): yield TEXT, result, pos elif isinstance(result, (int, float, long)): yield TEXT, number_conv(result), pos elif hasattr(result, '__iter__'): push(stream) stream = _ensure(result) break else: yield TEXT, unicode(result), pos elif kind is SUB: # This event is a list of directives and a list of nested # events to which those directives should be applied push(stream) stream = _apply_directives(data[1], data[0], ctxt, vars) break elif kind is EXEC: _exec_suite(data, ctxt, vars) else: yield kind, data, pos else: if not stack: break stream = pop()
def __call__(self, stream, ctxt=None, search_text=True): """Translate any localizable strings in the given stream. This function shouldn't be called directly. Instead, an instance of the `Translator` class should be registered as a filter with the `Template` or the `TemplateLoader`, or applied as a regular stream filter. If used as a template filter, it should be inserted in front of all the default filters. :param stream: the markup event stream :param ctxt: the template context (not used) :param search_text: whether text nodes should be translated (used internally) :return: the localized stream """ ignore_tags = self.ignore_tags include_attrs = self.include_attrs skip = 0 xml_lang = XML_NAMESPACE['lang'] if type(self.translate) is FunctionType: gettext = self.translate else: gettext = self.translate.ugettext if ctxt: ctxt['_i18n.gettext'] = gettext extract_text = self.extract_text if not extract_text: search_text = False for kind, data, pos in stream: # skip chunks that should not be localized if skip: if kind is START: skip += 1 elif kind is END: skip -= 1 yield kind, data, pos continue # handle different events that can be localized if kind is START: tag, attrs = data if tag in self.ignore_tags or \ isinstance(attrs.get(xml_lang), basestring): skip += 1 yield kind, data, pos continue new_attrs = [] changed = False for name, value in attrs: newval = value if extract_text and isinstance(value, basestring): if name in include_attrs: newval = gettext(value) else: newval = list(self(_ensure(value), ctxt, search_text=False) ) if newval != value: value = newval changed = True new_attrs.append((name, value)) if changed: attrs = Attrs(new_attrs) yield kind, (tag, attrs), pos elif search_text and kind is TEXT: text = data.strip() if text: data = data.replace(text, unicode(gettext(text))) yield kind, data, pos elif kind is SUB: directives, substream = data # If this is an i18n:msg directive, no need to translate text # nodes here is_msg = filter(None, [isinstance(d, MsgDirective) for d in directives]) substream = list(self(substream, ctxt, search_text=not is_msg)) yield kind, (directives, substream), pos else: yield kind, data, pos
def __call__(self, stream, ctxt=None, translate_text=True, translate_attrs=True): """Translate any localizable strings in the given stream. This function shouldn't be called directly. Instead, an instance of the `Translator` class should be registered as a filter with the `Template` or the `TemplateLoader`, or applied as a regular stream filter. If used as a template filter, it should be inserted in front of all the default filters. :param stream: the markup event stream :param ctxt: the template context (not used) :param translate_text: whether text nodes should be translated (used internally) :param translate_attrs: whether attribute values should be translated (used internally) :return: the localized stream """ ignore_tags = self.ignore_tags include_attrs = self.include_attrs skip = 0 xml_lang = XML_NAMESPACE['lang'] if not self.extract_text: translate_text = False translate_attrs = False if type(self.translate) is FunctionType: gettext = self.translate if ctxt: ctxt['_i18n.gettext'] = gettext else: if IS_PYTHON2: gettext = self.translate.ugettext ngettext = self.translate.ungettext else: gettext = self.translate.gettext ngettext = self.translate.ngettext try: if IS_PYTHON2: dgettext = self.translate.dugettext dngettext = self.translate.dungettext else: dgettext = self.translate.dgettext dngettext = self.translate.dngettext except AttributeError: dgettext = lambda _, y: gettext(y) dngettext = lambda _, s, p, n: ngettext(s, p, n) if ctxt: ctxt['_i18n.gettext'] = gettext ctxt['_i18n.ngettext'] = ngettext ctxt['_i18n.dgettext'] = dgettext ctxt['_i18n.dngettext'] = dngettext if ctxt and ctxt.get('_i18n.domain'): # TODO: This can cause infinite recursion if dgettext is defined # via the AttributeError case above! gettext = lambda msg: dgettext(ctxt.get('_i18n.domain'), msg) for kind, data, pos in stream: # skip chunks that should not be localized if skip: if kind is START: skip += 1 elif kind is END: skip -= 1 yield kind, data, pos continue # handle different events that can be localized if kind is START: tag, attrs = data if tag in self.ignore_tags or \ isinstance(attrs.get(xml_lang), basestring): skip += 1 yield kind, data, pos continue new_attrs = [] changed = False for name, value in attrs: newval = value if isinstance(value, basestring): if translate_attrs and name in include_attrs: newval = gettext(value) else: newval = list( self(_ensure(value), ctxt, translate_text=False)) if newval != value: value = newval changed = True new_attrs.append((name, value)) if changed: attrs = Attrs(new_attrs) yield kind, (tag, attrs), pos elif translate_text and kind is TEXT: text = data.strip() if text: data = data.replace(text, unicode(gettext(text))) yield kind, data, pos elif kind is SUB: directives, substream = data current_domain = None for idx, directive in enumerate(directives): # Organize directives to make everything work # FIXME: There's got to be a better way to do this! if isinstance(directive, DomainDirective): # Grab current domain and update context current_domain = directive.domain ctxt.push({'_i18n.domain': current_domain}) # Put domain directive as the first one in order to # update context before any other directives evaluation directives.insert(0, directives.pop(idx)) # If this is an i18n directive, no need to translate text # nodes here is_i18n_directive = any([ isinstance(d, ExtractableI18NDirective) for d in directives ]) substream = list( self(substream, ctxt, translate_text=not is_i18n_directive, translate_attrs=translate_attrs)) yield kind, (directives, substream), pos if current_domain: ctxt.pop() else: yield kind, data, pos
def extract(self, stream, gettext_functions=GETTEXT_FUNCTIONS, search_text=True, msgbuf=None): """Extract localizable strings from the given template stream. For every string found, this function yields a ``(lineno, function, message, comments)`` tuple, where: * ``lineno`` is the number of the line on which the string was found, * ``function`` is the name of the ``gettext`` function used (if the string was extracted from embedded Python code), and * ``message`` is the string itself (a ``unicode`` object, or a tuple of ``unicode`` objects for functions with multiple string arguments). * ``comments`` is a list of comments related to the message, extracted from ``i18n:comment`` attributes found in the markup >>> from genshi.template import MarkupTemplate >>> >>> tmpl = MarkupTemplate('''<html xmlns:py="http://genshi.edgewall.org/"> ... <head> ... <title>Example</title> ... </head> ... <body> ... <h1>Example</h1> ... <p>${_("Hello, %(name)s") % dict(name=username)}</p> ... <p>${ngettext("You have %d item", "You have %d items", num)}</p> ... </body> ... </html>''', filename='example.html') >>> >>> for line, func, msg, comments in Translator().extract(tmpl.stream): ... print "%d, %r, %r" % (line, func, msg) 3, None, u'Example' 6, None, u'Example' 7, '_', u'Hello, %(name)s' 8, 'ngettext', (u'You have %d item', u'You have %d items', None) :param stream: the event stream to extract strings from; can be a regular stream or a template stream :param gettext_functions: a sequence of function names that should be treated as gettext-style localization functions :param search_text: whether the content of text nodes should be extracted (used internally) :note: Changed in 0.4.1: For a function with multiple string arguments (such as ``ngettext``), a single item with a tuple of strings is yielded, instead an item for each string argument. :note: Changed in 0.6: The returned tuples now include a 4th element, which is a list of comments for the translator """ if not self.extract_text: search_text = False skip = 0 i18n_comment = I18N_NAMESPACE['comment'] i18n_msg = I18N_NAMESPACE['msg'] xml_lang = XML_NAMESPACE['lang'] for kind, data, pos in stream: if skip: if kind is START: skip += 1 if kind is END: skip -= 1 if kind is START and not skip: tag, attrs = data if tag in self.ignore_tags or \ isinstance(attrs.get(xml_lang), basestring): skip += 1 continue for name, value in attrs: if search_text and isinstance(value, basestring): if name in self.include_attrs: text = value.strip() if text: yield pos[1], None, text, [] else: for lineno, funcname, text, comments in self.extract( _ensure(value), gettext_functions, search_text=False): yield lineno, funcname, text, comments if msgbuf: msgbuf.append(kind, data, pos) else: msg_params = attrs.get(i18n_msg) if msg_params is not None: if type(msg_params) is list: # event tuple msg_params = msg_params[0][1] msgbuf = MessageBuffer( msg_params, attrs.get(i18n_comment), pos[1] ) elif not skip and search_text and kind is TEXT: if not msgbuf: text = data.strip() if text and filter(None, [ch.isalpha() for ch in text]): yield pos[1], None, text, [] else: msgbuf.append(kind, data, pos) elif not skip and msgbuf and kind is END: msgbuf.append(kind, data, pos) if not msgbuf.depth: yield msgbuf.lineno, None, msgbuf.format(), \ filter(None, [msgbuf.comment]) msgbuf = None elif kind is EXPR or kind is EXEC: if msgbuf: msgbuf.append(kind, data, pos) for funcname, strings in extract_from_code(data, gettext_functions): yield pos[1], funcname, strings, [] elif kind is SUB: subkind, substream = data messages = self.extract(substream, gettext_functions, search_text=search_text and not skip, msgbuf=msgbuf) for lineno, funcname, text, comments in messages: yield lineno, funcname, text, comments
def _flatten(self, stream, ctxt, **vars): number_conv = self._number_conv stack = [] push = stack.append pop = stack.pop stream = iter(stream) while 1: for kind, data, pos in stream: if kind is START and data[1]: # Attributes may still contain expressions in start tags at # this point, so do some evaluation tag, attrs = data new_attrs = [] for name, value in attrs: if type(value ) is list: # this is an interpolated string values = [ event[1] for event in self._flatten( value, ctxt, **vars) if event[0] is TEXT and event[1] is not None ] if not values: continue value = ''.join(values) new_attrs.append((name, value)) yield kind, (tag, Attrs(new_attrs)), pos elif kind is EXPR: result = _eval_expr(data, ctxt, vars) if result is not None: # First check for a string, otherwise the iterable test # below succeeds, and the string will be chopped up into # individual characters if isinstance(result, str): yield TEXT, result, pos elif isinstance(result, (int, float)): yield TEXT, number_conv(result), pos elif hasattr(result, '__iter__'): push(stream) stream = _ensure(result) break else: yield TEXT, str(result), pos elif kind is SUB: # This event is a list of directives and a list of nested # events to which those directives should be applied push(stream) stream = _apply_directives(data[1], data[0], ctxt, vars) break elif kind is EXEC: _exec_suite(data, ctxt, vars) else: yield kind, data, pos else: if not stack: break stream = pop()
def extract(self, stream, gettext_functions=GETTEXT_FUNCTIONS, search_text=True, msgbuf=None, comment_stack=None): """Extract localizable strings from the given template stream. For every string found, this function yields a ``(lineno, function, message, comments)`` tuple, where: * ``lineno`` is the number of the line on which the string was found, * ``function`` is the name of the ``gettext`` function used (if the string was extracted from embedded Python code), and * ``message`` is the string itself (a ``unicode`` object, or a tuple of ``unicode`` objects for functions with multiple string arguments). * ``comments`` is a list of comments related to the message, extracted from ``i18n:comment`` attributes found in the markup >>> tmpl = MarkupTemplate('''<html xmlns:py="http://genshi.edgewall.org/"> ... <head> ... <title>Example</title> ... </head> ... <body> ... <h1>Example</h1> ... <p>${_("Hello, %(name)s") % dict(name=username)}</p> ... <p>${ngettext("You have %d item", "You have %d items", num)}</p> ... </body> ... </html>''', filename='example.html') >>> for line, func, msg, comments in Translator().extract(tmpl.stream): ... print('%d, %r, %r' % (line, func, msg)) 3, None, u'Example' 6, None, u'Example' 7, '_', u'Hello, %(name)s' 8, 'ngettext', (u'You have %d item', u'You have %d items', None) :param stream: the event stream to extract strings from; can be a regular stream or a template stream :param gettext_functions: a sequence of function names that should be treated as gettext-style localization functions :param search_text: whether the content of text nodes should be extracted (used internally) :note: Changed in 0.4.1: For a function with multiple string arguments (such as ``ngettext``), a single item with a tuple of strings is yielded, instead an item for each string argument. :note: Changed in 0.6: The returned tuples now include a fourth element, which is a list of comments for the translator. """ if not self.extract_text: search_text = False if comment_stack is None: comment_stack = [] skip = 0 # Un-comment bellow to extract messages without adding directives xml_lang = XML_NAMESPACE['lang'] for kind, data, pos in stream: if skip: if kind is START: skip += 1 if kind is END: skip -= 1 if kind is START and not skip: tag, attrs = data if tag in self.ignore_tags or \ isinstance(attrs.get(xml_lang), basestring): skip += 1 continue for name, value in attrs: if search_text and isinstance(value, basestring): if name in self.include_attrs: text = value.strip() if text: # XXX: Do we need to grab i18n:comment from comment_stack ??? yield pos[1], None, text, [] else: for lineno, funcname, text, comments in self.extract( _ensure(value), gettext_functions, search_text=False): yield lineno, funcname, text, comments if msgbuf: msgbuf.append(kind, data, pos) elif not skip and search_text and kind is TEXT: if not msgbuf: text = data.strip() if text and [ch for ch in text if ch.isalpha()]: yield pos[1], None, text, comment_stack[-1:] else: msgbuf.append(kind, data, pos) elif not skip and msgbuf and kind is END: msgbuf.append(kind, data, pos) if not msgbuf.depth: yield msgbuf.lineno, None, msgbuf.format(), [ c for c in msgbuf.comment if c ] msgbuf = None elif kind is EXPR or kind is EXEC: if msgbuf: msgbuf.append(kind, data, pos) for funcname, strings in extract_from_code( data, gettext_functions): # XXX: Do we need to grab i18n:comment from comment_stack ??? yield pos[1], funcname, strings, [] elif kind is SUB: directives, substream = data in_comment = False for idx, directive in enumerate(directives): # Do a first loop to see if there's a comment directive # If there is update context and pop it from directives if isinstance(directive, CommentDirective): in_comment = True comment_stack.append(directive.comment) if len(directives) == 1: # in case we're in the presence of something like: # <p i18n:comment="foo">Foo</p> messages = self.extract( substream, gettext_functions, search_text=search_text and not skip, msgbuf=msgbuf, comment_stack=comment_stack) for lineno, funcname, text, comments in messages: yield lineno, funcname, text, comments directives.pop(idx) elif not isinstance(directive, I18NDirective): # Remove all other non i18n directives from the process directives.pop(idx) if not directives and not in_comment: # Extract content if there's no directives because # strip was pop'ed and not because comment was pop'ed. # Extraction in this case has been taken care of. messages = self.extract(substream, gettext_functions, search_text=search_text and not skip, msgbuf=msgbuf) for lineno, funcname, text, comments in messages: yield lineno, funcname, text, comments for directive in directives: if isinstance(directive, ExtractableI18NDirective): messages = directive.extract(substream, comment_stack) for funcname, text, comments in messages: yield pos[1], funcname, text, comments else: messages = self.extract(substream, gettext_functions, search_text=search_text and not skip, msgbuf=msgbuf) for lineno, funcname, text, comments in messages: yield lineno, funcname, text, comments if in_comment: comment_stack.pop()