def test_dynamic_msgids(self):
sample_source = """
<p i18n:translate="">
Some
<span tal:replace="string:strange">dynamic</span>
text.
</p>
<p i18n:translate="">
A <a tal:attributes="href path:dynamic">link</a>.
</p>
"""
p = HTMLTALParser()
p.parseString(sample_source)
program, macros = p.getCode()
engine = POEngine()
engine.file = 'sample_source'
POTALInterpreter(program,
macros,
engine,
stream=StringIO(),
metal=False)()
msgids = []
for domain in engine.catalog.values():
msgids += list(domain)
msgids.sort()
self.assertEqual(msgids, [
'A <a href="${DYNAMIC_CONTENT}">link</a>.',
'Some ${DYNAMIC_CONTENT} text.'
])
def test_dynamic_msgids(self):
sample_source = """
<p i18n:translate="">
Some
<span tal:replace="string:strange">dynamic</span>
text.
</p>
<p i18n:translate="">
A <a tal:attributes="href path:dynamic">link</a>.
</p>
"""
p = HTMLTALParser()
p.parseString(sample_source)
program, macros = p.getCode()
engine = POEngine()
engine.file = 'sample_source'
POTALInterpreter(program, macros, engine, stream=StringIO(),
metal=False)()
msgids = []
for domain in engine.catalog.values():
msgids += list(domain)
msgids.sort()
self.assertEqual(msgids,
['A <a href="${DYNAMIC_CONTENT}">link</a>.',
'Some ${DYNAMIC_CONTENT} text.'])
def test_translate(self):
test_keys = ['foo', 'bar', 'blarf', 'washington']
engine = POEngine()
engine.file = 'foo.pt'
for key in test_keys:
engine.translate(key, 'domain')
for key in test_keys:
self.assertTrue(key in engine.catalog['domain'],
"POEngine catalog does not properly store message ids"
)
def test_translate(self):
test_keys = ['foo', 'bar', 'blarf', 'washington']
engine = POEngine()
engine.file = 'foo.pt'
for key in test_keys:
engine.translate(key, 'domain')
for key in test_keys:
self.assertIn(
key, engine.catalog['domain'],
"POEngine catalog does not properly store message ids")
def tal_strings(dir, domain="zope", include_default_domain=False, exclude=()):
"""Retrieve all TAL messages from `dir` that are in the `domain`.
"""
# We import zope.tal.talgettext here because we can't rely on the
# right sys path until app_dir has run
from zope.tal.talgettext import POEngine, POTALInterpreter
from zope.tal.htmltalparser import HTMLTALParser
from zope.tal.talparser import TALParser
import sys
# i18n patch if templates using special characters
reload(sys)
sys.setdefaultencoding('UTF8')
engine = POEngine()
class Devnull(object):
def write(self, s):
pass
for filename in (find_files(dir, '*.*pt', exclude=tuple(exclude)) +
find_files(dir, '*.html', exclude=tuple(exclude)) +
find_files(dir, '*.kupu', exclude=tuple(exclude)) +
find_files(dir, '*.pox', exclude=tuple(exclude)) +
find_files(dir, '*.xsl', exclude=tuple(exclude))):
try:
engine.file = filename
name, ext = os.path.splitext(filename)
if ext == '.html' or ext.endswith('pt'):
p = HTMLTALParser()
else:
p = TALParser()
p.parseFile(filename)
program, macros = p.getCode()
POTALInterpreter(program, macros, engine, stream=Devnull(),
metal=False)()
except: # Hee hee, I love bare excepts!
print 'There was an error processing', filename
traceback.print_exc()
# See whether anything in the domain was found
if not domain in engine.catalog:
return {}
# We do not want column numbers.
catalog = engine.catalog[domain].copy()
# When the Domain is 'default', then this means that none was found;
# Include these strings; yes or no?
if include_default_domain:
catalog.update(engine.catalog['default'])
for msgid, locations in catalog.items():
catalog[msgid] = [(l[0], l[1][0]) for l in locations]
return catalog
def tal_strings(dir, domain="zope", include_default_domain=False, exclude=()):
"""Retrieve all TAL messages from `dir` that are in the `domain`.
"""
# We import zope.tal.talgettext here because we can't rely on the
# right sys path until app_dir has run
from zope.tal.talgettext import POEngine, POTALInterpreter
from zope.tal.htmltalparser import HTMLTALParser
from zope.tal.talparser import TALParser
engine = POEngine()
class Devnull(object):
def write(self, s):
pass
for filename in (find_files(dir, '*.*pt', exclude=tuple(exclude)) +
find_files(dir, '*.html', exclude=tuple(exclude)) +
find_files(dir, '*.kupu', exclude=tuple(exclude)) +
find_files(dir, '*.pox', exclude=tuple(exclude)) +
find_files(dir, '*.xsl', exclude=tuple(exclude))):
try:
engine.file = filename
name, ext = os.path.splitext(filename)
if ext == '.html' or ext.endswith('pt'):
p = HTMLTALParser()
else:
p = TALParser()
p.parseFile(filename)
program, macros = p.getCode()
POTALInterpreter(program,
macros,
engine,
stream=Devnull(),
metal=False)()
except: # Hee hee, I love bare excepts!
print 'There was an error processing', filename
traceback.print_exc()
# See whether anything in the domain was found
if not domain in engine.catalog:
return {}
# We do not want column numbers.
catalog = engine.catalog[domain].copy()
# When the Domain is 'default', then this means that none was found;
# Include these strings; yes or no?
if include_default_domain:
catalog.update(engine.catalog['default'])
for msgid, locations in catalog.items():
catalog[msgid] = [(l[0], l[1][0]) for l in locations]
return catalog
def test_potalinterpreter_translate_default(self):
sample_source = '<p i18n:translate="">text</p>'
p = HTMLTALParser()
p.parseString(sample_source)
program, macros = p.getCode()
engine = POEngine()
engine.file = 'sample_source'
interpreter = POTALInterpreter(
program, macros, engine, stream=StringIO(), metal=False)
# We simply call this, to make sure we don't get a NameError
# for 'unicode' in python 3.
# The return value (strangely: 'x') is not interesting here.
interpreter.translate('text')
msgids = []
for domain in engine.catalog.values():
msgids += list(domain)
self.assertIn('text', msgids)
def test_translate_existing(self):
engine = POEngine()
# This tries to reproduce a big surfacing in a template of
# PloneSoftwareCenter when using the i18ndude package to
# extract translatable strings, which uses zope.tal. The
# relevant html snippet is this:
#
# <a href="#" title="Read more…"
# i18n:attributes="title label_read_more"
# tal:attributes="href release/absolute_url">
# <span i18n:translate="label_read_more">Read more…</span>
# </a>
#
# Due to the different ways that i18n:attributes and
# i18n:translate are handled, the attribute gets passed to the
# translate method with the html entity interpreted as a
# unicode, and the i18n:translate gets passed as a simple
# string with the html entity intact. That may need a fix
# elsewhere, but at the moment it gives a warning. The very
# least we can do is make sure that this does not give a
# UnicodeDecodeError, which is what we test here.
engine.file = 'psc_release_listing.pt'
# position is position in file.
engine.translate('foo',
'domain',
default=u'Read more\u2026',
position=7)
# Adding the same key with the same default is fine.
engine.translate('foo',
'domain',
default=u'Read more\u2026',
position=13)
# Adding the same key with a different default is bad and
# triggers a warning.
with warnings.catch_warnings(record=True) as log:
warnings.simplefilter("always")
engine.translate('foo',
'domain',
default='Read still more…',
position=42)
self.assertEqual(len(log), 1)
message = log[0].message
with tempfile.TemporaryFile('w+') as printfile:
print(message, file=printfile)
printfile.seek(0)
self.assertTrue("already exists with a different default" in
printfile.read())
def test_potalinterpreter_translate_default(self):
sample_source = '<p i18n:translate="">text</p>'
p = HTMLTALParser()
p.parseString(sample_source)
program, macros = p.getCode()
engine = POEngine()
engine.file = 'sample_source'
interpreter = POTALInterpreter(program,
macros,
engine,
stream=StringIO(),
metal=False)
# We simply call this, to make sure we don't get a NameError
# for 'unicode' in python 3.
# The return value (strangely: 'x') is not interesting here.
interpreter.translate('text')
msgids = []
for domain in engine.catalog.values():
msgids += list(domain)
self.assertIn('text', msgids)
def tal_strings(dir, domain="zope", include_default_domain=False):
"""Retrieve all TAL messages from `dir` that are in the `domain`.
"""
# We import zope.tal.talgettext here because we can't rely on the
# right sys path until app_dir has run
from zope.tal.talgettext import POEngine, POTALInterpreter
from zope.tal.htmltalparser import HTMLTALParser
engine = POEngine()
class Devnull(object):
def write(self, s):
pass
for filename in find_files(dir, '*.pt'):
try:
engine.file = filename
p = HTMLTALParser()
p.parseFile(filename)
program, macros = p.getCode()
POTALInterpreter(program, macros, engine, stream=Devnull(),
metal=False)()
except: # Hee hee, I love bare excepts!
print 'There was an error processing', filename
traceback.print_exc()
# See whether anything in the domain was found
if not engine.catalog.has_key(domain):
return {}
# We do not want column numbers.
catalog = engine.catalog[domain].copy()
# When the Domain is 'default', then this means that none was found;
# Include these strings; yes or no?
if include_default_domain:
catalog.update(engine.catalog['zope'])
for msgid, locations in catalog.items():
catalog[msgid] = map(lambda l: (l[0], l[1][0]), locations)
return catalog
def test_translate_existing(self):
engine = POEngine()
# This tries to reproduce a big surfacing in a template of
# PloneSoftwareCenter when using the i18ndude package to
# extract translatable strings, which uses zope.tal. The
# relevant html snippet is this:
#
# <a href="#" title="Read more…"
# i18n:attributes="title label_read_more"
# tal:attributes="href release/absolute_url">
# <span i18n:translate="label_read_more">Read more…</span>
# </a>
#
# Due to the different ways that i18n:attributes and
# i18n:translate are handled, the attribute gets passed to the
# translate method with the html entity interpreted as a
# unicode, and the i18n:translate gets passed as a simple
# string with the html entity intact. That may need a fix
# elsewhere, but at the moment it gives a warning. The very
# least we can do is make sure that this does not give a
# UnicodeDecodeError, which is what we test here.
engine.file = 'psc_release_listing.pt'
# position is position in file.
engine.translate('foo', 'domain',
default=u'Read more\u2026', position=7)
# Adding the same key with the same default is fine.
engine.translate('foo', 'domain',
default=u'Read more\u2026', position=13)
# Adding the same key with a different default is bad and
# triggers a warning.
with warnings.catch_warnings(record=True) as log:
warnings.simplefilter("always")
engine.translate('foo', 'domain',
default='Read still more…', position=42)
self.assertEqual(len(log), 1)
message = log[0].message
with tempfile.TemporaryFile('w+') as printfile:
print(message, file=printfile)
printfile.seek(0)
self.assertTrue("already exists with a different default"
in printfile.read())
def tal_strings(dir,
domain="zope",
include_default_domain=False,
exclude=(),
filePattern='*.pt'):
"""Retrieve all TAL messages from `dir` that are in the `domain`.
>>> from zope.app.locales import extract
>>> import tempfile
>>> dir = tempfile.mkdtemp()
Let's create a page template in the i18n domain ``test``:
>>> testpt = open(os.path.join(dir, 'test.pt'), 'w')
>>> testpt.write('<tal:block i18n:domain="test" i18n:translate="">test</tal:block>')
>>> testpt.close()
And now one in no domain:
>>> nopt = open(os.path.join(dir, 'no.pt'), 'w')
>>> nopt.write('<tal:block i18n:translate="">no domain</tal:block>')
>>> nopt.close()
Now let's find the strings for the domain ``test``:
>>> extract.tal_strings(dir, domain='test', include_default_domain=True)
{u'test': [('...test.pt', 1)], u'no domain': [('...no.pt', 1)]}
And now an xml file
>>> xml = open(os.path.join(dir, 'xml.pt'), 'w')
>>> xml.write('''<?xml version="1.0" encoding="utf-8"?>
... <rss version="2.0"
... i18n:domain="xml"
... xmlns:i18n="http://xml.zope.org/namespaces/i18n"
... xmlns:tal="http://xml.zope.org/namespaces/tal"
... xmlns="http://purl.org/rss/1.0/modules/content/">
... <channel>
... <link i18n:translate="">Link Content</link>
... </channel>
... </rss>
... ''')
>>> xml.close()
>>> extract.tal_strings(dir, domain='xml')
{u'Link Content': [('...xml.pt', 8)]}
We also provide a file with a different file ending:
>>> testpt = open(os.path.join(dir, 'test.html'), 'w')
>>> testpt.write('<tal:block i18n:domain="html" i18n:translate="">html</tal:block>')
>>> testpt.close()
>>> extract.tal_strings(dir, domain='html', include_default_domain=True,
... filePattern='*.html')
{u'html': [('...test.html', 1)]}
Cleanup
>>> import shutil
>>> shutil.rmtree(dir)
"""
# We import zope.tal.talgettext here because we can't rely on the
# right sys path until app_dir has run
from zope.tal.talgettext import POEngine, POTALInterpreter
from zope.tal.htmltalparser import HTMLTALParser
from zope.tal.talparser import TALParser
engine = POEngine()
class Devnull(object):
def write(self, s):
pass
for filename in find_files(dir, filePattern, exclude=tuple(exclude)):
f = file(filename, 'rb')
start = f.read(6)
f.close()
if start.startswith('<?xml'):
parserFactory = TALParser
else:
parserFactory = HTMLTALParser
try:
engine.file = filename
p = parserFactory()
p.parseFile(filename)
program, macros = p.getCode()
POTALInterpreter(program,
macros,
engine,
stream=Devnull(),
metal=False)()
except: # Hee hee, I love bare excepts!
print 'There was an error processing', filename
traceback.print_exc()
# See whether anything in the domain was found
if not engine.catalog.has_key(domain):
return {}
# We do not want column numbers.
catalog = engine.catalog[domain].copy()
# When the Domain is 'default', then this means that none was found;
# Include these strings; yes or no?
if include_default_domain:
defaultCatalog = engine.catalog.get('default')
if defaultCatalog == None:
engine.catalog['default'] = {}
catalog.update(engine.catalog['default'])
for msgid, locations in catalog.items():
catalog[msgid] = map(lambda l: (l[0], l[1][0]), locations)
return catalog
def tal_strings(dir, domain="zope", include_default_domain=False, exclude=()):
"""Retrieve all TAL messages from `dir` that are in the `domain`.
"""
# We import zope.tal.talgettext here because we can't rely on the
# right sys path until app_dir has run
from zope.pagetemplate.pagetemplatefile import sniff_type
from zope.pagetemplate.pagetemplatefile import XML_PREFIX_MAX_LENGTH
from zope.tal.talgettext import POEngine, POTALInterpreter
from zope.tal.htmltalparser import HTMLTALParser
from zope.tal.talparser import TALParser
engine = POEngine()
class Devnull(object):
def write(self, s):
pass
filenames = find_files(dir, '*.pt', exclude=tuple(exclude)) \
+ find_files(dir, '*.html', exclude=tuple(exclude)) \
+ find_files(dir, '*.xml', exclude=tuple(exclude))
for filename in sorted(filenames):
# This is taken from zope/pagetemplate/pagetemplatefile.py (r40504)
f = open(filename, "rb")
try:
text = f.read(XML_PREFIX_MAX_LENGTH)
except:
f.close()
raise
type_ = sniff_type(text)
if type_ == "text/xml":
text += f.read()
else:
# For HTML, we really want the file read in text mode:
f.close()
f = open(filename)
text = f.read()
f.close()
try:
engine.file = filename
if type_ != "text/xml":
p = HTMLTALParser()
else:
p = TALParser()
p.parseString(text)
program, macros = p.getCode()
POTALInterpreter(program, macros, engine, stream=Devnull(),
metal=False)()
except: # Hee hee, I love bare excepts!
print 'There was an error processing', filename
traceback.print_exc()
# See whether anything in the domain was found
if not engine.catalog.has_key(domain):
return {}
# We do not want column numbers.
catalog = engine.catalog[domain].copy()
# When the Domain is 'default', then this means that none was found;
# Include these strings; yes or no?
if include_default_domain:
catalog.update(engine.catalog['zope'])
for msgid, locations in catalog.items():
catalog[msgid] = map(lambda l: (l[0], l[1][0]), locations)
return catalog
def tal_strings(dir, domain="zope", include_default_domain=False, exclude=()):
"""Retrieve all TAL messages from `dir` that are in the `domain`.
"""
# We import zope.tal.talgettext here because we can't rely on the
# right sys path until app_dir has run
from zope.pagetemplate.pagetemplatefile import sniff_type
from zope.pagetemplate.pagetemplatefile import XML_PREFIX_MAX_LENGTH
from zope.tal.talgettext import POEngine, POTALInterpreter
from zope.tal.htmltalparser import HTMLTALParser
from zope.tal.talparser import TALParser
engine = POEngine()
class Devnull(object):
def write(self, s):
pass
filenames = find_files(dir, '*.pt', exclude=tuple(exclude)) \
+ find_files(dir, '*.html', exclude=tuple(exclude)) \
+ find_files(dir, '*.xml', exclude=tuple(exclude))
for filename in sorted(filenames):
# This is taken from zope/pagetemplate/pagetemplatefile.py (r40504)
f = open(filename, "rb")
try:
text = f.read(XML_PREFIX_MAX_LENGTH)
except:
f.close()
raise
type_ = sniff_type(text)
if type_ == "text/xml":
text += f.read()
else:
# For HTML, we really want the file read in text mode:
f.close()
f = open(filename)
text = f.read()
f.close()
try:
engine.file = filename
if type_ != "text/xml":
p = HTMLTALParser()
else:
p = TALParser()
p.parseString(text)
program, macros = p.getCode()
POTALInterpreter(program,
macros,
engine,
stream=Devnull(),
metal=False)()
except: # Hee hee, I love bare excepts!
print 'There was an error processing', filename
traceback.print_exc()
# See whether anything in the domain was found
if not engine.catalog.has_key(domain):
return {}
# We do not want column numbers.
catalog = engine.catalog[domain].copy()
# When the Domain is 'default', then this means that none was found;
# Include these strings; yes or no?
if include_default_domain:
defaultCatalog = engine.catalog.get('default')
if defaultCatalog == None:
engine.catalog['default'] = {}
catalog.update(engine.catalog['default'])
for msgid, locations in catalog.items():
catalog[msgid] = map(lambda l: (l[0], l[1][0]), locations)
return catalog
def tal_strings(dir,
domain="zope",
include_default_domain=False,
exclude=(),
filePattern='*.pt'):
"""Retrieve all TAL messages from `dir` that are in the `domain`.
>>> from zope.app.locales import extract
>>> import tempfile
>>> dir = tempfile.mkdtemp()
Let's create a page template in the i18n domain ``test``:
>>> testpt = open(os.path.join(dir, 'test.pt'), 'w')
>>> testpt.write('<tal:block i18n:domain="test" i18n:translate="">test</tal:block>')
>>> testpt.close()
And now one in no domain:
>>> nopt = open(os.path.join(dir, 'no.pt'), 'w')
>>> nopt.write('<tal:block i18n:translate="">no domain</tal:block>')
>>> nopt.close()
Now let's find the strings for the domain ``test``:
>>> extract.tal_strings(dir, domain='test', include_default_domain=True)
{u'test': [('...test.pt', 1)], u'no domain': [('...no.pt', 1)]}
And now an xml file
>>> xml = open(os.path.join(dir, 'xml.pt'), 'w')
>>> xml.write('''<?xml version="1.0" encoding="utf-8"?>
... <rss version="2.0"
... i18n:domain="xml"
... xmlns:i18n="http://xml.zope.org/namespaces/i18n"
... xmlns:tal="http://xml.zope.org/namespaces/tal"
... xmlns="http://purl.org/rss/1.0/modules/content/">
... <channel>
... <link i18n:translate="">Link Content</link>
... </channel>
... </rss>
... ''')
>>> xml.close()
>>> extract.tal_strings(dir, domain='xml')
{u'Link Content': [('...xml.pt', 8)]}
We also provide a file with a different file ending:
>>> testpt = open(os.path.join(dir, 'test.html'), 'w')
>>> testpt.write('<tal:block i18n:domain="html" i18n:translate="">html</tal:block>')
>>> testpt.close()
>>> extract.tal_strings(dir, domain='html', include_default_domain=True,
... filePattern='*.html')
{u'html': [('...test.html', 1)]}
Cleanup
>>> import shutil
>>> shutil.rmtree(dir)
"""
# We import zope.tal.talgettext here because we can't rely on the
# right sys path until app_dir has run
from zope.tal.talgettext import POEngine, POTALInterpreter
from zope.tal.htmltalparser import HTMLTALParser
from zope.tal.talparser import TALParser
engine = POEngine()
class Devnull(object):
def write(self, s):
pass
for filename in find_files(dir, filePattern, exclude=tuple(exclude)):
f = file(filename,'rb')
start = f.read(6)
f.close()
if start.startswith('<?xml'):
parserFactory = TALParser
else:
parserFactory = HTMLTALParser
try:
engine.file = filename
p = parserFactory()
p.parseFile(filename)
program, macros = p.getCode()
POTALInterpreter(program, macros, engine, stream=Devnull(),
metal=False)()
except: # Hee hee, I love bare excepts!
print 'There was an error processing', filename
traceback.print_exc()
# See whether anything in the domain was found
if not engine.catalog.has_key(domain):
return {}
# We do not want column numbers.
catalog = engine.catalog[domain].copy()
# When the Domain is 'default', then this means that none was found;
# Include these strings; yes or no?
if include_default_domain:
defaultCatalog = engine.catalog.get('default')
if defaultCatalog == None:
engine.catalog['default'] = {}
catalog.update(engine.catalog['default'])
for msgid, locations in catalog.items():
catalog[msgid] = map(lambda l: (l[0], l[1][0]), locations)
return catalog
def tal_strings(dir, domain="zope", include_default_domain=False, exclude=()):
"""Retrieve all TAL messages from `dir` that are in the `domain`.
"""
# We import zope.tal.talgettext here because we can't rely on the
# right sys path until app_dir has run
from zope.tal.talgettext import POEngine, POTALInterpreter
from zope.tal.htmltalparser import HTMLTALParser
from zope.tal.talparser import TALParser
engine = POEngine()
class Devnull(object):
def write(self, s):
pass
for filename in (find_files(dir, '*.*pt', exclude=tuple(exclude)) +
find_files(dir, '*.html', exclude=tuple(exclude)) +
find_files(dir, '*.kupu', exclude=tuple(exclude)) +
find_files(dir, '*.pox', exclude=tuple(exclude)) +
find_files(dir, '*.xsl', exclude=tuple(exclude))):
engine.file = filename
name, ext = os.path.splitext(filename)
# First try with standard zope.tal parsers.
if ext == '.html' or ext.endswith('pt'):
parser = HTMLTALParser()
else:
parser = TALParser()
try:
parser.parseFile(filename)
program, macros = parser.getCode()
POTALInterpreter(program, macros, engine, stream=Devnull(),
metal=False)()
except KeyboardInterrupt:
raise
except: # Hee hee, I love bare excepts!
if ext == '.html' or ext.endswith('pt'):
# We can have one retry with our own generator.
gen = DudeGenerator(xml=0)
parser = HTMLTALParser(gen=gen)
try:
parser.parseFile(filename)
program, macros = parser.getCode()
POTALInterpreter(program, macros, engine, stream=Devnull(),
metal=False)()
except: # Hee hee, I love bare excepts!
print 'There was an error processing', filename
traceback.print_exc()
else:
print 'There was an error processing', filename
traceback.print_exc()
# See whether anything in the domain was found
if domain not in engine.catalog:
return {}
# We do not want column numbers.
catalog = engine.catalog[domain].copy()
# When the Domain is 'default', then this means that none was found;
# Include these strings; yes or no?
if include_default_domain:
catalog.update(engine.catalog['default'])
for msgid, locations in catalog.items():
catalog[msgid] = [(l[0], l[1][0]) for l in locations]
return catalog