def _extract_attrs(self, event, gettext_functions, search_text):
for name, value in event[1][1]:
if search_text and isinstance(value, basestring):
if name in self.include_attrs:
text = value.strip()
if text:
yield event[2][1], None, text, []
else:
for message in self.extract(_ensure(value), gettext_functions, search_text=False):
yield message
def _extract_attrs(self, event, gettext_functions, search_text):
for name, value in event[1][1]:
if search_text and isinstance(value, basestring):
if name in self.include_attrs:
text = value.strip()
if text:
yield event[2][1], None, text, []
else:
for message in self.extract(_ensure(value), gettext_functions,
search_text=False):
yield message
def _eval(self, stream, ctxt, **vars):
"""Internal stream filter that evaluates any expressions in `START` and
`TEXT` events.
"""
filters = (self._flatten, self._eval)
number_conv = self._number_conv
for kind, data, pos in stream:
if kind is START and data[1]:
# Attributes may still contain expressions in start tags at
# this point, so do some evaluation
tag, attrs = data
new_attrs = []
for name, substream in attrs:
if isinstance(substream, basestring):
value = substream
else:
values = []
for subkind, subdata, subpos in self._eval(substream,
ctxt,
**vars):
if subkind is TEXT:
values.append(subdata)
value = [x for x in values if x is not None]
if not value:
continue
new_attrs.append((name, u''.join(value)))
yield kind, (tag, Attrs(new_attrs)), pos
elif kind is EXPR:
result = _eval_expr(data, ctxt, **vars)
if result is not None:
# First check for a string, otherwise the iterable test
# below succeeds, and the string will be chopped up into
# individual characters
if isinstance(result, basestring):
yield TEXT, result, pos
elif isinstance(result, (int, float, long)):
yield TEXT, number_conv(result), pos
elif hasattr(result, '__iter__'):
substream = _ensure(result)
for filter_ in filters:
substream = filter_(substream, ctxt, **vars)
for event in substream:
yield event
else:
yield TEXT, unicode(result), pos
else:
yield kind, data, pos
def output (self, template, **kwargs):
""" Output the page. """
# Send HTTP content-type header.
cherrypy.response.headers['Content-Type'] = self.CONTENT_TYPE
template = Formatters.formatters[self.FORMATTER].templates[template]
ctxt = genshi.template.Context (cherrypy = cherrypy, bs = BaseSearcher, **kwargs)
stream = template.stream
for filter_ in template.filters:
stream = filter_ (iter (stream), ctxt)
serializer = self.get_serializer ()
return genshi.output.encode (serializer (
_ensure (genshi.Stream (stream))), encoding = 'utf-8')
def render(self, page, os):
""" Render and send to browser. """
self.send_headers()
template = self.templates[page]
ctxt = genshi.template.Context(cherrypy=cherrypy,
os=os,
bs=BaseSearcher)
stream = template.stream
for filter_ in template.filters:
stream = filter_(iter(stream), ctxt)
# there's no easy way in genshi to pass collapse_lines to this filter
stream = WHITESPACE_FILTER(stream, collapse_lines=COLLAPSE_LINES)
return genshi.output.encode(self.get_serializer()(_ensure(
genshi.Stream(stream))),
encoding='utf-8')
def __call__(self, stream, ctxt=None, translate_text=True,
translate_attrs=True):
"""Translate any localizable strings in the given stream.
This function shouldn't be called directly. Instead, an instance of
the `Translator` class should be registered as a filter with the
`Template` or the `TemplateLoader`, or applied as a regular stream
filter. If used as a template filter, it should be inserted in front of
all the default filters.
:param stream: the markup event stream
:param ctxt: the template context (not used)
:param translate_text: whether text nodes should be translated (used
internally)
:param translate_attrs: whether attribute values should be translated
(used internally)
:return: the localized stream
"""
ignore_tags = self.ignore_tags
include_attrs = self.include_attrs
skip = 0
xml_lang = XML_NAMESPACE['lang']
if not self.extract_text:
translate_text = False
translate_attrs = False
if type(self.translate) is FunctionType:
gettext = self.translate
if ctxt:
ctxt['_i18n.gettext'] = gettext
else:
if IS_PYTHON2:
gettext = self.translate.ugettext
ngettext = self.translate.ungettext
else:
gettext = self.translate.gettext
ngettext = self.translate.ngettext
try:
if IS_PYTHON2:
dgettext = self.translate.dugettext
dngettext = self.translate.dungettext
else:
dgettext = self.translate.dgettext
dngettext = self.translate.dngettext
except AttributeError:
dgettext = lambda _, y: gettext(y)
dngettext = lambda _, s, p, n: ngettext(s, p, n)
if ctxt:
ctxt['_i18n.gettext'] = gettext
ctxt['_i18n.ngettext'] = ngettext
ctxt['_i18n.dgettext'] = dgettext
ctxt['_i18n.dngettext'] = dngettext
if ctxt and ctxt.get('_i18n.domain'):
# TODO: This can cause infinite recursion if dgettext is defined
# via the AttributeError case above!
gettext = lambda msg: dgettext(ctxt.get('_i18n.domain'), msg)
for kind, data, pos in stream:
# skip chunks that should not be localized
if skip:
if kind is START:
skip += 1
elif kind is END:
skip -= 1
yield kind, data, pos
continue
# handle different events that can be localized
if kind is START:
tag, attrs = data
if tag in self.ignore_tags or \
isinstance(attrs.get(xml_lang), basestring):
skip += 1
yield kind, data, pos
continue
new_attrs = []
changed = False
for name, value in attrs:
newval = value
if isinstance(value, basestring):
if translate_attrs and name in include_attrs:
newval = gettext(value)
else:
newval = list(
self(_ensure(value), ctxt, translate_text=False)
)
if newval != value:
value = newval
changed = True
new_attrs.append((name, value))
if changed:
attrs = Attrs(new_attrs)
yield kind, (tag, attrs), pos
elif translate_text and kind is TEXT:
text = data.strip()
if text:
data = data.replace(text, unicode(gettext(text)))
yield kind, data, pos
elif kind is SUB:
directives, substream = data
current_domain = None
for idx, directive in enumerate(directives):
# Organize directives to make everything work
# FIXME: There's got to be a better way to do this!
if isinstance(directive, DomainDirective):
# Grab current domain and update context
current_domain = directive.domain
ctxt.push({'_i18n.domain': current_domain})
# Put domain directive as the first one in order to
# update context before any other directives evaluation
directives.insert(0, directives.pop(idx))
# If this is an i18n directive, no need to translate text
# nodes here
is_i18n_directive = any([
isinstance(d, ExtractableI18NDirective)
for d in directives
])
substream = list(self(substream, ctxt,
translate_text=not is_i18n_directive,
translate_attrs=translate_attrs))
yield kind, (directives, substream), pos
if current_domain:
ctxt.pop()
else:
yield kind, data, pos
def extract(self,
stream,
gettext_functions=GETTEXT_FUNCTIONS,
search_text=True):
"""Extract localizable strings from the given template stream.
For every string found, this function yields a ``(lineno, function,
message)`` tuple, where:
* ``lineno`` is the number of the line on which the string was found,
* ``function`` is the name of the ``gettext`` function used (if the
string was extracted from embedded Python code), and
* ``message`` is the string itself (a ``unicode`` object, or a tuple
of ``unicode`` objects for functions with multiple string arguments).
>>> from genshi.template import MarkupTemplate
>>>
>>> tmpl = MarkupTemplate('''<html xmlns:py="http://genshi.edgewall.org/">
... <head>
... <title>Example</title>
... </head>
... <body>
... <h1>Example</h1>
... <p>${_("Hello, %(name)s") % dict(name=username)}</p>
... <p>${ngettext("You have %d item", "You have %d items", num)}</p>
... </body>
... </html>''', filename='example.html')
>>>
>>> for lineno, funcname, message in Translator().extract(tmpl.stream):
... print "%d, %r, %r" % (lineno, funcname, message)
3, None, u'Example'
6, None, u'Example'
7, '_', u'Hello, %(name)s'
8, 'ngettext', (u'You have %d item', u'You have %d items')
:param stream: the event stream to extract strings from; can be a
regular stream or a template stream
:param gettext_functions: a sequence of function names that should be
treated as gettext-style localization
functions
:param search_text: whether the content of text nodes should be
extracted (used internally)
:note: Changed in 0.4.1: For a function with multiple string arguments
(such as ``ngettext``), a single item with a tuple of strings is
yielded, instead an item for each string argument.
"""
tagname = None
if not self.extract_text:
search_text = False
skip = 0
xml_lang = XML_NAMESPACE['lang']
for kind, data, pos in stream:
if skip:
if kind is START:
skip += 1
if kind is END:
skip -= 1
if kind is START and not skip:
tag, attrs = data
if tag in self.ignore_tags or \
isinstance(attrs.get(xml_lang), basestring):
skip += 1
continue
for name, value in attrs:
if search_text and isinstance(value, basestring):
if name in self.include_attrs:
text = value.strip()
if text:
yield pos[1], None, text
else:
for lineno, funcname, text in self.extract(
_ensure(value),
gettext_functions,
search_text=False):
yield lineno, funcname, text
elif not skip and search_text and kind is TEXT:
text = data.strip()
if text and filter(None, [ch.isalpha() for ch in text]):
yield pos[1], None, text
elif kind is EXPR or kind is EXEC:
consts = dict([(n, chr(i) + '\x00')
for i, n in enumerate(data.code.co_consts)])
gettext_locs = [
consts[n] for n in gettext_functions if n in consts
]
ops = [
_LOAD_CONST, '(', '|'.join(gettext_locs), ')',
_CALL_FUNCTION, '.\x00', '((?:', _BINARY_ADD, '|',
_LOAD_CONST, '.\x00)+)'
]
for loc, opcodes in re.findall(''.join(ops),
data.code.co_code):
funcname = data.code.co_consts[ord(loc[0])]
strings = []
opcodes = iter(opcodes)
for opcode in opcodes:
if opcode == _BINARY_ADD:
arg = strings.pop()
strings[-1] += arg
else:
arg = data.code.co_consts[ord(opcodes.next())]
opcodes.next() # skip second byte
if not isinstance(arg, basestring):
break
strings.append(unicode(arg))
if len(strings) == 1:
strings = strings[0]
else:
strings = tuple(strings)
yield pos[1], funcname, strings
elif kind is SUB:
subkind, substream = data
messages = self.extract(substream,
gettext_functions,
search_text=search_text and not skip)
for lineno, funcname, text in messages:
yield lineno, funcname, text
def __call__(self, stream, ctxt=None, search_text=True):
"""Translate any localizable strings in the given stream.
This function shouldn't be called directly. Instead, an instance of
the `Translator` class should be registered as a filter with the
`Template` or the `TemplateLoader`, or applied as a regular stream
filter. If used as a template filter, it should be inserted in front of
all the default filters.
:param stream: the markup event stream
:param ctxt: the template context (not used)
:param search_text: whether text nodes should be translated (used
internally)
:return: the localized stream
"""
ignore_tags = self.ignore_tags
include_attrs = self.include_attrs
translate = self.translate
if not self.extract_text:
search_text = False
skip = 0
xml_lang = XML_NAMESPACE['lang']
for kind, data, pos in stream:
# skip chunks that should not be localized
if skip:
if kind is START:
skip += 1
elif kind is END:
skip -= 1
yield kind, data, pos
continue
# handle different events that can be localized
if kind is START:
tag, attrs = data
if tag in self.ignore_tags or \
isinstance(attrs.get(xml_lang), basestring):
skip += 1
yield kind, data, pos
continue
new_attrs = []
changed = False
for name, value in attrs:
newval = value
if search_text and isinstance(value, basestring):
if name in include_attrs:
newval = self.translate(value)
else:
newval = list(
self(_ensure(value), ctxt, search_text=False))
if newval != value:
value = newval
changed = True
new_attrs.append((name, value))
if changed:
attrs = new_attrs
yield kind, (tag, attrs), pos
elif search_text and kind is TEXT:
text = data.strip()
if text:
data = data.replace(text, translate(text))
yield kind, data, pos
elif kind is SUB:
subkind, substream = data
new_substream = list(self(substream, ctxt))
yield kind, (subkind, new_substream), pos
else:
yield kind, data, pos
def extract(self, stream, gettext_functions=GETTEXT_FUNCTIONS,
search_text=True, msgbuf=None, comment_stack=None):
"""Extract localizable strings from the given template stream.
For every string found, this function yields a ``(lineno, function,
message, comments)`` tuple, where:
* ``lineno`` is the number of the line on which the string was found,
* ``function`` is the name of the ``gettext`` function used (if the
string was extracted from embedded Python code), and
* ``message`` is the string itself (a ``unicode`` object, or a tuple
of ``unicode`` objects for functions with multiple string
arguments).
* ``comments`` is a list of comments related to the message, extracted
from ``i18n:comment`` attributes found in the markup
>>> tmpl = MarkupTemplate('''<html xmlns:py="http://genshi.edgewall.org/">
... <head>
... <title>Example</title>
... </head>
... <body>
... <h1>Example</h1>
... <p>${_("Hello, %(name)s") % dict(name=username)}</p>
... <p>${ngettext("You have %d item", "You have %d items", num)}</p>
... </body>
... </html>''', filename='example.html')
>>> for line, func, msg, comments in Translator().extract(tmpl.stream):
... print('%d, %r, %r' % (line, func, msg))
3, None, u'Example'
6, None, u'Example'
7, '_', u'Hello, %(name)s'
8, 'ngettext', (u'You have %d item', u'You have %d items', None)
:param stream: the event stream to extract strings from; can be a
regular stream or a template stream
:param gettext_functions: a sequence of function names that should be
treated as gettext-style localization
functions
:param search_text: whether the content of text nodes should be
extracted (used internally)
:note: Changed in 0.4.1: For a function with multiple string arguments
(such as ``ngettext``), a single item with a tuple of strings is
yielded, instead an item for each string argument.
:note: Changed in 0.6: The returned tuples now include a fourth
element, which is a list of comments for the translator.
"""
if not self.extract_text:
search_text = False
if comment_stack is None:
comment_stack = []
skip = 0
# Un-comment bellow to extract messages without adding directives
xml_lang = XML_NAMESPACE['lang']
for kind, data, pos in stream:
if skip:
if kind is START:
skip += 1
if kind is END:
skip -= 1
if kind is START and not skip:
tag, attrs = data
if tag in self.ignore_tags or \
isinstance(attrs.get(xml_lang), basestring):
skip += 1
continue
for name, value in attrs:
if search_text and isinstance(value, basestring):
if name in self.include_attrs:
text = value.strip()
if text:
# XXX: Do we need to grab i18n:comment from comment_stack ???
yield pos[1], None, text, []
else:
for lineno, funcname, text, comments in self.extract(
_ensure(value), gettext_functions,
search_text=False):
yield lineno, funcname, text, comments
if msgbuf:
msgbuf.append(kind, data, pos)
elif not skip and search_text and kind is TEXT:
if not msgbuf:
text = data.strip()
if text and [ch for ch in text if ch.isalpha()]:
yield pos[1], None, text, comment_stack[-1:]
else:
msgbuf.append(kind, data, pos)
elif not skip and msgbuf and kind is END:
msgbuf.append(kind, data, pos)
if not msgbuf.depth:
yield msgbuf.lineno, None, msgbuf.format(), [
c for c in msgbuf.comment if c
]
msgbuf = None
elif kind is EXPR or kind is EXEC:
if msgbuf:
msgbuf.append(kind, data, pos)
for funcname, strings in extract_from_code(data,
gettext_functions):
# XXX: Do we need to grab i18n:comment from comment_stack ???
yield pos[1], funcname, strings, []
elif kind is SUB:
directives, substream = data
in_comment = False
for idx, directive in enumerate(directives):
# Do a first loop to see if there's a comment directive
# If there is update context and pop it from directives
if isinstance(directive, CommentDirective):
in_comment = True
comment_stack.append(directive.comment)
if len(directives) == 1:
# in case we're in the presence of something like:
# <p i18n:comment="foo">Foo</p>
messages = self.extract(
substream, gettext_functions,
search_text=search_text and not skip,
msgbuf=msgbuf, comment_stack=comment_stack)
for lineno, funcname, text, comments in messages:
yield lineno, funcname, text, comments
directives.pop(idx)
elif not isinstance(directive, I18NDirective):
# Remove all other non i18n directives from the process
directives.pop(idx)
if not directives and not in_comment:
# Extract content if there's no directives because
# strip was pop'ed and not because comment was pop'ed.
# Extraction in this case has been taken care of.
messages = self.extract(
substream, gettext_functions,
search_text=search_text and not skip, msgbuf=msgbuf)
for lineno, funcname, text, comments in messages:
yield lineno, funcname, text, comments
for directive in directives:
if isinstance(directive, ExtractableI18NDirective):
messages = directive.extract(substream, comment_stack)
for funcname, text, comments in messages:
yield pos[1], funcname, text, comments
else:
messages = self.extract(
substream, gettext_functions,
search_text=search_text and not skip, msgbuf=msgbuf)
for lineno, funcname, text, comments in messages:
yield lineno, funcname, text, comments
if in_comment:
comment_stack.pop()
def _inject(self):
content = self.content
if hasattr(content, '__call__'):
content = content()
for event in _ensure(content):
yield None, event
def __call__(self, stream):
"""Apply the transform filter to the marked stream.
:param stream: The marked event stream to filter
"""
found_first_th = False
if self.data['xhr']:
in_dirlist = True # XHR rows are only the interesting table
else:
in_dirlist = False
in_repoindex = in_name_td = False
idx = 0
rows_seen = 0
for kind, data, pos in stream:
if kind == START:
if all((data[0] == QName("http://www.w3.org/1999/xhtml}table"),
data[1].get('id') == 'dirlist',
self.data['dir'])):
in_dirlist = True
in_repoindex = data[1].get('id') == 'repoindex'
if all((in_dirlist, not found_first_th,
data[0] == QName("http://www.w3.org/1999/xhtml}th"))):
for event in _ensure(tag.th(Markup(' '))):
yield event
found_first_th = True
yield kind, data, pos
continue
if in_dirlist and data[0] == QName("http://www.w3.org/1999/xhtml}td"):
rows_seen = rows_seen + 1
if rows_seen == 1 and ('up' in self.data['chrome']['links'] or \
(self.data['dir'] and not self.data['dir']['entries'])):
data = data[0], data[1] - 'colspan' | [(QName('colspan'), '7')]
yield kind, data, pos
continue # don't mess with the "parent link" and "No files found" row
if data[1].get('class') == 'name':
# would be nice to get this static for any particular
# item. We can't use a simple offset count due
# to the XHR requests
uid = uuid.uuid4()
for event in _ensure(tag.td(tag.input(type='checkbox',
id="cb-%s" % uid,
class_='fileselect'))):
yield event
in_name_td = True
elif in_dirlist and kind == END and data == QName("http://www.w3.org/1999/xhtml}table"):
# we're leaving the current table; reset markers
in_dirlist = False
rows_seen = 0
found_first_th = False
idx = 0
elif in_name_td and kind == END:
if data == QName("http://www.w3.org/1999/xhtml}td"):
in_name_td = False
elif data == QName("http://www.w3.org/1999/xhtml}a"):
yield kind, data, pos
if idx == 0 and in_repoindex:
# Don't yield a context menu for the repos since they don't have a dir entry
continue
entry = self.data['dir']['entries'][idx]
menu = tag.div(tag.a(class_="ctx-expander fa fa-angle-down"),
tag.div(
tag.ul(tag.li(tag.span("%s (version %d)" % (entry.name, entry.rev),
class_="plain-text")),
class_="styled-dropdown"),
class_="bottom-fix"
),
id="ctx-%s" % uid,
class_="inline-block margin-left-small dropdown-toggle")
for provider in sorted(self.context_menu_providers, key=lambda x: x.get_order(self.req)):
content = provider.get_content(self.req, entry, self.data)
if content:
menu.children[1].children[0].append(tag.li(content))
if len(menu.children[1].children[0].children) > 1:
for event in _ensure(menu):
yield event
idx = idx + 1
continue
yield kind, data, pos
def extract(self,
stream,
gettext_functions=GETTEXT_FUNCTIONS,
search_text=True,
msgbuf=None):
"""Extract localizable strings from the given template stream.
For every string found, this function yields a ``(lineno, function,
message, comments)`` tuple, where:
* ``lineno`` is the number of the line on which the string was found,
* ``function`` is the name of the ``gettext`` function used (if the
string was extracted from embedded Python code), and
* ``message`` is the string itself (a ``unicode`` object, or a tuple
of ``unicode`` objects for functions with multiple string
arguments).
* ``comments`` is a list of comments related to the message, extracted
from ``i18n:comment`` attributes found in the markup
>>> from genshi.template import MarkupTemplate
>>>
>>> tmpl = MarkupTemplate('''<html xmlns:py="http://genshi.edgewall.org/">
... <head>
... <title>Example</title>
... </head>
... <body>
... <h1>Example</h1>
... <p>${_("Hello, %(name)s") % dict(name=username)}</p>
... <p>${ngettext("You have %d item", "You have %d items", num)}</p>
... </body>
... </html>''', filename='example.html')
>>>
>>> for line, func, msg, comments in Translator().extract(tmpl.stream):
... print "%d, %r, %r" % (line, func, msg)
3, None, u'Example'
6, None, u'Example'
7, '_', u'Hello, %(name)s'
8, 'ngettext', (u'You have %d item', u'You have %d items', None)
:param stream: the event stream to extract strings from; can be a
regular stream or a template stream
:param gettext_functions: a sequence of function names that should be
treated as gettext-style localization
functions
:param search_text: whether the content of text nodes should be
extracted (used internally)
:note: Changed in 0.4.1: For a function with multiple string arguments
(such as ``ngettext``), a single item with a tuple of strings is
yielded, instead an item for each string argument.
:note: Changed in 0.6: The returned tuples now include a 4th element,
which is a list of comments for the translator
"""
if not self.extract_text:
search_text = False
skip = 0
i18n_comment = I18N_NAMESPACE['comment']
i18n_msg = I18N_NAMESPACE['msg']
xml_lang = XML_NAMESPACE['lang']
for kind, data, pos in stream:
if skip:
if kind is START:
skip += 1
if kind is END:
skip -= 1
if kind is START and not skip:
tag, attrs = data
if tag in self.ignore_tags or \
isinstance(attrs.get(xml_lang), basestring):
skip += 1
continue
for name, value in attrs:
if search_text and isinstance(value, basestring):
if name in self.include_attrs:
text = value.strip()
if text:
yield pos[1], None, text, []
else:
for lineno, funcname, text, comments in self.extract(
_ensure(value),
gettext_functions,
search_text=False):
yield lineno, funcname, text, comments
if msgbuf:
msgbuf.append(kind, data, pos)
else:
msg_params = attrs.get(i18n_msg)
if msg_params is not None:
if type(msg_params) is list: # event tuple
msg_params = msg_params[0][1]
msgbuf = MessageBuffer(msg_params,
attrs.get(i18n_comment), pos[1])
elif not skip and search_text and kind is TEXT:
if not msgbuf:
text = data.strip()
if text and filter(None, [ch.isalpha() for ch in text]):
yield pos[1], None, text, []
else:
msgbuf.append(kind, data, pos)
elif not skip and msgbuf and kind is END:
msgbuf.append(kind, data, pos)
if not msgbuf.depth:
yield msgbuf.lineno, None, msgbuf.format(), \
filter(None, [msgbuf.comment])
msgbuf = None
elif kind is EXPR or kind is EXEC:
if msgbuf:
msgbuf.append(kind, data, pos)
for funcname, strings in extract_from_code(
data, gettext_functions):
yield pos[1], funcname, strings, []
elif kind is SUB:
subkind, substream = data
messages = self.extract(substream,
gettext_functions,
search_text=search_text and not skip,
msgbuf=msgbuf)
for lineno, funcname, text, comments in messages:
yield lineno, funcname, text, comments
def __call__(self, stream, ctxt=None, search_text=True, msgbuf=None):
"""Translate any localizable strings in the given stream.
This function shouldn't be called directly. Instead, an instance of
the `Translator` class should be registered as a filter with the
`Template` or the `TemplateLoader`, or applied as a regular stream
filter. If used as a template filter, it should be inserted in front of
all the default filters.
:param stream: the markup event stream
:param ctxt: the template context (not used)
:param search_text: whether text nodes should be translated (used
internally)
:param msgbuf: a `MessageBuffer` object or `None` (used internally)
:return: the localized stream
"""
ignore_tags = self.ignore_tags
include_attrs = self.include_attrs
translate = self.translate
if not self.extract_text:
search_text = False
skip = 0
i18n_msg = I18N_NAMESPACE['msg']
ns_prefixes = []
xml_lang = XML_NAMESPACE['lang']
for kind, data, pos in stream:
# skip chunks that should not be localized
if skip:
if kind is START:
skip += 1
elif kind is END:
skip -= 1
yield kind, data, pos
continue
# handle different events that can be localized
if kind is START:
tag, attrs = data
if tag in self.ignore_tags or \
isinstance(attrs.get(xml_lang), basestring):
skip += 1
yield kind, data, pos
continue
new_attrs = []
changed = False
for name, value in attrs:
newval = value
if search_text and isinstance(value, basestring):
if name in include_attrs:
newval = self.translate(value)
else:
newval = list(self(_ensure(value), ctxt,
search_text=False, msgbuf=msgbuf)
)
if newval != value:
value = newval
changed = True
new_attrs.append((name, value))
if changed:
attrs = Attrs(new_attrs)
if msgbuf:
msgbuf.append(kind, data, pos)
continue
elif i18n_msg in attrs:
params = attrs.get(i18n_msg)
if params and type(params) is list: # event tuple
params = params[0][1]
msgbuf = MessageBuffer(params)
attrs -= i18n_msg
yield kind, (tag, attrs), pos
elif search_text and kind is TEXT:
if not msgbuf:
text = data.strip()
if text:
data = data.replace(text, unicode(translate(text)))
yield kind, data, pos
else:
msgbuf.append(kind, data, pos)
elif msgbuf and kind is EXPR:
msgbuf.append(kind, data, pos)
elif not skip and msgbuf and kind is END:
msgbuf.append(kind, data, pos)
if not msgbuf.depth:
for event in msgbuf.translate(translate(msgbuf.format())):
yield event
msgbuf = None
yield kind, data, pos
elif kind is SUB:
subkind, substream = data
new_substream = list(self(substream, ctxt, msgbuf=msgbuf))
yield kind, (subkind, new_substream), pos
elif kind is START_NS and data[1] == I18N_NAMESPACE:
ns_prefixes.append(data[0])
elif kind is END_NS and data in ns_prefixes:
ns_prefixes.remove(data)
else:
yield kind, data, pos
def _inject(self):
content = self.content
if callable(content):
content = content()
for event in _ensure(content):
yield None, event
def _flatten(self, stream, ctxt, **vars):
number_conv = self._number_conv
stack = []
push = stack.append
pop = stack.pop
stream = iter(stream)
while 1:
for kind, data, pos in stream:
if kind is START and data[1]:
# Attributes may still contain expressions in start tags at
# this point, so do some evaluation
tag, attrs = data
new_attrs = []
for name, value in attrs:
if type(value) is list: # this is an interpolated string
values = [event[1]
for event in self._flatten(value, ctxt, **vars)
if event[0] is TEXT and event[1] is not None
]
if not values:
continue
value = ''.join(values)
new_attrs.append((name, value))
yield kind, (tag, Attrs(new_attrs)), pos
elif kind is EXPR:
result = _eval_expr(data, ctxt, vars)
if result is not None:
# First check for a string, otherwise the iterable test
# below succeeds, and the string will be chopped up into
# individual characters
if isinstance(result, basestring):
yield TEXT, result, pos
elif isinstance(result, (int, float, long)):
yield TEXT, number_conv(result), pos
elif hasattr(result, '__iter__'):
push(stream)
stream = _ensure(result)
break
else:
yield TEXT, unicode(result), pos
elif kind is SUB:
# This event is a list of directives and a list of nested
# events to which those directives should be applied
push(stream)
stream = _apply_directives(data[1], data[0], ctxt, vars)
break
elif kind is EXEC:
_exec_suite(data, ctxt, vars)
else:
yield kind, data, pos
else:
if not stack:
break
stream = pop()
def _inject(self):
content = self.content
if hasattr(content, '__call__'):
content = content()
for event in _ensure(content):
yield None, event
def __call__(self, stream, ctxt=None, search_text=True):
"""Translate any localizable strings in the given stream.
This function shouldn't be called directly. Instead, an instance of
the `Translator` class should be registered as a filter with the
`Template` or the `TemplateLoader`, or applied as a regular stream
filter. If used as a template filter, it should be inserted in front of
all the default filters.
:param stream: the markup event stream
:param ctxt: the template context (not used)
:param search_text: whether text nodes should be translated (used
internally)
:return: the localized stream
"""
ignore_tags = self.ignore_tags
include_attrs = self.include_attrs
skip = 0
xml_lang = XML_NAMESPACE['lang']
if type(self.translate) is FunctionType:
gettext = self.translate
else:
gettext = self.translate.ugettext
if ctxt:
ctxt['_i18n.gettext'] = gettext
extract_text = self.extract_text
if not extract_text:
search_text = False
for kind, data, pos in stream:
# skip chunks that should not be localized
if skip:
if kind is START:
skip += 1
elif kind is END:
skip -= 1
yield kind, data, pos
continue
# handle different events that can be localized
if kind is START:
tag, attrs = data
if tag in self.ignore_tags or \
isinstance(attrs.get(xml_lang), basestring):
skip += 1
yield kind, data, pos
continue
new_attrs = []
changed = False
for name, value in attrs:
newval = value
if extract_text and isinstance(value, basestring):
if name in include_attrs:
newval = gettext(value)
else:
newval = list(self(_ensure(value), ctxt,
search_text=False)
)
if newval != value:
value = newval
changed = True
new_attrs.append((name, value))
if changed:
attrs = Attrs(new_attrs)
yield kind, (tag, attrs), pos
elif search_text and kind is TEXT:
text = data.strip()
if text:
data = data.replace(text, unicode(gettext(text)))
yield kind, data, pos
elif kind is SUB:
directives, substream = data
# If this is an i18n:msg directive, no need to translate text
# nodes here
is_msg = filter(None, [isinstance(d, MsgDirective)
for d in directives])
substream = list(self(substream, ctxt,
search_text=not is_msg))
yield kind, (directives, substream), pos
else:
yield kind, data, pos
def __call__(self,
stream,
ctxt=None,
translate_text=True,
translate_attrs=True):
"""Translate any localizable strings in the given stream.
This function shouldn't be called directly. Instead, an instance of
the `Translator` class should be registered as a filter with the
`Template` or the `TemplateLoader`, or applied as a regular stream
filter. If used as a template filter, it should be inserted in front of
all the default filters.
:param stream: the markup event stream
:param ctxt: the template context (not used)
:param translate_text: whether text nodes should be translated (used
internally)
:param translate_attrs: whether attribute values should be translated
(used internally)
:return: the localized stream
"""
ignore_tags = self.ignore_tags
include_attrs = self.include_attrs
skip = 0
xml_lang = XML_NAMESPACE['lang']
if not self.extract_text:
translate_text = False
translate_attrs = False
if type(self.translate) is FunctionType:
gettext = self.translate
if ctxt:
ctxt['_i18n.gettext'] = gettext
else:
if IS_PYTHON2:
gettext = self.translate.ugettext
ngettext = self.translate.ungettext
else:
gettext = self.translate.gettext
ngettext = self.translate.ngettext
try:
if IS_PYTHON2:
dgettext = self.translate.dugettext
dngettext = self.translate.dungettext
else:
dgettext = self.translate.dgettext
dngettext = self.translate.dngettext
except AttributeError:
dgettext = lambda _, y: gettext(y)
dngettext = lambda _, s, p, n: ngettext(s, p, n)
if ctxt:
ctxt['_i18n.gettext'] = gettext
ctxt['_i18n.ngettext'] = ngettext
ctxt['_i18n.dgettext'] = dgettext
ctxt['_i18n.dngettext'] = dngettext
if ctxt and ctxt.get('_i18n.domain'):
# TODO: This can cause infinite recursion if dgettext is defined
# via the AttributeError case above!
gettext = lambda msg: dgettext(ctxt.get('_i18n.domain'), msg)
for kind, data, pos in stream:
# skip chunks that should not be localized
if skip:
if kind is START:
skip += 1
elif kind is END:
skip -= 1
yield kind, data, pos
continue
# handle different events that can be localized
if kind is START:
tag, attrs = data
if tag in self.ignore_tags or \
isinstance(attrs.get(xml_lang), basestring):
skip += 1
yield kind, data, pos
continue
new_attrs = []
changed = False
for name, value in attrs:
newval = value
if isinstance(value, basestring):
if translate_attrs and name in include_attrs:
newval = gettext(value)
else:
newval = list(
self(_ensure(value), ctxt, translate_text=False))
if newval != value:
value = newval
changed = True
new_attrs.append((name, value))
if changed:
attrs = Attrs(new_attrs)
yield kind, (tag, attrs), pos
elif translate_text and kind is TEXT:
text = data.strip()
if text:
data = data.replace(text, unicode(gettext(text)))
yield kind, data, pos
elif kind is SUB:
directives, substream = data
current_domain = None
for idx, directive in enumerate(directives):
# Organize directives to make everything work
# FIXME: There's got to be a better way to do this!
if isinstance(directive, DomainDirective):
# Grab current domain and update context
current_domain = directive.domain
ctxt.push({'_i18n.domain': current_domain})
# Put domain directive as the first one in order to
# update context before any other directives evaluation
directives.insert(0, directives.pop(idx))
# If this is an i18n directive, no need to translate text
# nodes here
is_i18n_directive = any([
isinstance(d, ExtractableI18NDirective) for d in directives
])
substream = list(
self(substream,
ctxt,
translate_text=not is_i18n_directive,
translate_attrs=translate_attrs))
yield kind, (directives, substream), pos
if current_domain:
ctxt.pop()
else:
yield kind, data, pos
def extract(self, stream, gettext_functions=GETTEXT_FUNCTIONS,
search_text=True, msgbuf=None):
"""Extract localizable strings from the given template stream.
For every string found, this function yields a ``(lineno, function,
message, comments)`` tuple, where:
* ``lineno`` is the number of the line on which the string was found,
* ``function`` is the name of the ``gettext`` function used (if the
string was extracted from embedded Python code), and
* ``message`` is the string itself (a ``unicode`` object, or a tuple
of ``unicode`` objects for functions with multiple string
arguments).
* ``comments`` is a list of comments related to the message, extracted
from ``i18n:comment`` attributes found in the markup
>>> from genshi.template import MarkupTemplate
>>>
>>> tmpl = MarkupTemplate('''<html xmlns:py="http://genshi.edgewall.org/">
... <head>
... <title>Example</title>
... </head>
... <body>
... <h1>Example</h1>
... <p>${_("Hello, %(name)s") % dict(name=username)}</p>
... <p>${ngettext("You have %d item", "You have %d items", num)}</p>
... </body>
... </html>''', filename='example.html')
>>>
>>> for line, func, msg, comments in Translator().extract(tmpl.stream):
... print "%d, %r, %r" % (line, func, msg)
3, None, u'Example'
6, None, u'Example'
7, '_', u'Hello, %(name)s'
8, 'ngettext', (u'You have %d item', u'You have %d items', None)
:param stream: the event stream to extract strings from; can be a
regular stream or a template stream
:param gettext_functions: a sequence of function names that should be
treated as gettext-style localization
functions
:param search_text: whether the content of text nodes should be
extracted (used internally)
:note: Changed in 0.4.1: For a function with multiple string arguments
(such as ``ngettext``), a single item with a tuple of strings is
yielded, instead an item for each string argument.
:note: Changed in 0.6: The returned tuples now include a 4th element,
which is a list of comments for the translator
"""
if not self.extract_text:
search_text = False
skip = 0
i18n_comment = I18N_NAMESPACE['comment']
i18n_msg = I18N_NAMESPACE['msg']
xml_lang = XML_NAMESPACE['lang']
for kind, data, pos in stream:
if skip:
if kind is START:
skip += 1
if kind is END:
skip -= 1
if kind is START and not skip:
tag, attrs = data
if tag in self.ignore_tags or \
isinstance(attrs.get(xml_lang), basestring):
skip += 1
continue
for name, value in attrs:
if search_text and isinstance(value, basestring):
if name in self.include_attrs:
text = value.strip()
if text:
yield pos[1], None, text, []
else:
for lineno, funcname, text, comments in self.extract(
_ensure(value), gettext_functions,
search_text=False):
yield lineno, funcname, text, comments
if msgbuf:
msgbuf.append(kind, data, pos)
else:
msg_params = attrs.get(i18n_msg)
if msg_params is not None:
if type(msg_params) is list: # event tuple
msg_params = msg_params[0][1]
msgbuf = MessageBuffer(
msg_params, attrs.get(i18n_comment), pos[1]
)
elif not skip and search_text and kind is TEXT:
if not msgbuf:
text = data.strip()
if text and filter(None, [ch.isalpha() for ch in text]):
yield pos[1], None, text, []
else:
msgbuf.append(kind, data, pos)
elif not skip and msgbuf and kind is END:
msgbuf.append(kind, data, pos)
if not msgbuf.depth:
yield msgbuf.lineno, None, msgbuf.format(), \
filter(None, [msgbuf.comment])
msgbuf = None
elif kind is EXPR or kind is EXEC:
if msgbuf:
msgbuf.append(kind, data, pos)
for funcname, strings in extract_from_code(data,
gettext_functions):
yield pos[1], funcname, strings, []
elif kind is SUB:
subkind, substream = data
messages = self.extract(substream, gettext_functions,
search_text=search_text and not skip,
msgbuf=msgbuf)
for lineno, funcname, text, comments in messages:
yield lineno, funcname, text, comments
def _flatten(self, stream, ctxt, **vars):
number_conv = self._number_conv
stack = []
push = stack.append
pop = stack.pop
stream = iter(stream)
while 1:
for kind, data, pos in stream:
if kind is START and data[1]:
# Attributes may still contain expressions in start tags at
# this point, so do some evaluation
tag, attrs = data
new_attrs = []
for name, value in attrs:
if type(value
) is list: # this is an interpolated string
values = [
event[1] for event in self._flatten(
value, ctxt, **vars)
if event[0] is TEXT and event[1] is not None
]
if not values:
continue
value = ''.join(values)
new_attrs.append((name, value))
yield kind, (tag, Attrs(new_attrs)), pos
elif kind is EXPR:
result = _eval_expr(data, ctxt, vars)
if result is not None:
# First check for a string, otherwise the iterable test
# below succeeds, and the string will be chopped up into
# individual characters
if isinstance(result, str):
yield TEXT, result, pos
elif isinstance(result, (int, float)):
yield TEXT, number_conv(result), pos
elif hasattr(result, '__iter__'):
push(stream)
stream = _ensure(result)
break
else:
yield TEXT, str(result), pos
elif kind is SUB:
# This event is a list of directives and a list of nested
# events to which those directives should be applied
push(stream)
stream = _apply_directives(data[1], data[0], ctxt, vars)
break
elif kind is EXEC:
_exec_suite(data, ctxt, vars)
else:
yield kind, data, pos
else:
if not stack:
break
stream = pop()
def extract(self,
stream,
gettext_functions=GETTEXT_FUNCTIONS,
search_text=True,
msgbuf=None,
comment_stack=None):
"""Extract localizable strings from the given template stream.
For every string found, this function yields a ``(lineno, function,
message, comments)`` tuple, where:
* ``lineno`` is the number of the line on which the string was found,
* ``function`` is the name of the ``gettext`` function used (if the
string was extracted from embedded Python code), and
* ``message`` is the string itself (a ``unicode`` object, or a tuple
of ``unicode`` objects for functions with multiple string
arguments).
* ``comments`` is a list of comments related to the message, extracted
from ``i18n:comment`` attributes found in the markup
>>> tmpl = MarkupTemplate('''<html xmlns:py="http://genshi.edgewall.org/">
... <head>
... <title>Example</title>
... </head>
... <body>
... <h1>Example</h1>
... <p>${_("Hello, %(name)s") % dict(name=username)}</p>
... <p>${ngettext("You have %d item", "You have %d items", num)}</p>
... </body>
... </html>''', filename='example.html')
>>> for line, func, msg, comments in Translator().extract(tmpl.stream):
... print('%d, %r, %r' % (line, func, msg))
3, None, u'Example'
6, None, u'Example'
7, '_', u'Hello, %(name)s'
8, 'ngettext', (u'You have %d item', u'You have %d items', None)
:param stream: the event stream to extract strings from; can be a
regular stream or a template stream
:param gettext_functions: a sequence of function names that should be
treated as gettext-style localization
functions
:param search_text: whether the content of text nodes should be
extracted (used internally)
:note: Changed in 0.4.1: For a function with multiple string arguments
(such as ``ngettext``), a single item with a tuple of strings is
yielded, instead an item for each string argument.
:note: Changed in 0.6: The returned tuples now include a fourth
element, which is a list of comments for the translator.
"""
if not self.extract_text:
search_text = False
if comment_stack is None:
comment_stack = []
skip = 0
# Un-comment bellow to extract messages without adding directives
xml_lang = XML_NAMESPACE['lang']
for kind, data, pos in stream:
if skip:
if kind is START:
skip += 1
if kind is END:
skip -= 1
if kind is START and not skip:
tag, attrs = data
if tag in self.ignore_tags or \
isinstance(attrs.get(xml_lang), basestring):
skip += 1
continue
for name, value in attrs:
if search_text and isinstance(value, basestring):
if name in self.include_attrs:
text = value.strip()
if text:
# XXX: Do we need to grab i18n:comment from comment_stack ???
yield pos[1], None, text, []
else:
for lineno, funcname, text, comments in self.extract(
_ensure(value),
gettext_functions,
search_text=False):
yield lineno, funcname, text, comments
if msgbuf:
msgbuf.append(kind, data, pos)
elif not skip and search_text and kind is TEXT:
if not msgbuf:
text = data.strip()
if text and [ch for ch in text if ch.isalpha()]:
yield pos[1], None, text, comment_stack[-1:]
else:
msgbuf.append(kind, data, pos)
elif not skip and msgbuf and kind is END:
msgbuf.append(kind, data, pos)
if not msgbuf.depth:
yield msgbuf.lineno, None, msgbuf.format(), [
c for c in msgbuf.comment if c
]
msgbuf = None
elif kind is EXPR or kind is EXEC:
if msgbuf:
msgbuf.append(kind, data, pos)
for funcname, strings in extract_from_code(
data, gettext_functions):
# XXX: Do we need to grab i18n:comment from comment_stack ???
yield pos[1], funcname, strings, []
elif kind is SUB:
directives, substream = data
in_comment = False
for idx, directive in enumerate(directives):
# Do a first loop to see if there's a comment directive
# If there is update context and pop it from directives
if isinstance(directive, CommentDirective):
in_comment = True
comment_stack.append(directive.comment)
if len(directives) == 1:
# in case we're in the presence of something like:
# <p i18n:comment="foo">Foo</p>
messages = self.extract(
substream,
gettext_functions,
search_text=search_text and not skip,
msgbuf=msgbuf,
comment_stack=comment_stack)
for lineno, funcname, text, comments in messages:
yield lineno, funcname, text, comments
directives.pop(idx)
elif not isinstance(directive, I18NDirective):
# Remove all other non i18n directives from the process
directives.pop(idx)
if not directives and not in_comment:
# Extract content if there's no directives because
# strip was pop'ed and not because comment was pop'ed.
# Extraction in this case has been taken care of.
messages = self.extract(substream,
gettext_functions,
search_text=search_text
and not skip,
msgbuf=msgbuf)
for lineno, funcname, text, comments in messages:
yield lineno, funcname, text, comments
for directive in directives:
if isinstance(directive, ExtractableI18NDirective):
messages = directive.extract(substream, comment_stack)
for funcname, text, comments in messages:
yield pos[1], funcname, text, comments
else:
messages = self.extract(substream,
gettext_functions,
search_text=search_text
and not skip,
msgbuf=msgbuf)
for lineno, funcname, text, comments in messages:
yield lineno, funcname, text, comments
if in_comment:
comment_stack.pop()
