def test_po_with_multiline_obsolete_message(self):
catalog = Catalog()
catalog.add(u'foo', u'Voh', locations=[('main.py', 1)])
msgid = r"""Here's a message that covers
multiple lines, and should still be handled
correctly.
"""
msgstr = r"""Here's a message that covers
multiple lines, and should still be handled
correctly.
"""
catalog.obsolete[msgid] = Message(msgid, msgstr,
locations=[('utils.py', 3)])
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=True)
self.assertEqual(b'''#: main.py:1
msgid "foo"
msgstr "Voh"
#~ msgid ""
#~ "Here's a message that covers\\n"
#~ "multiple lines, and should still be handled\\n"
#~ "correctly.\\n"
#~ msgstr ""
#~ "Here's a message that covers\\n"
#~ "multiple lines, and should still be handled\\n"
#~ "correctly.\\n"''', buf.getvalue().strip())
def test_no_wrap_and_width_behaviour_on_comments(self):
catalog = Catalog()
catalog.add("Pretty dam long message id, which must really be big "
"to test this wrap behaviour, if not it won't work.",
locations=[("fake.py", n) for n in range(1, 30)])
buf = BytesIO()
pofile.write_po(buf, catalog, width=None, omit_header=True)
self.assertEqual(b"""\
#: fake.py:1 fake.py:2 fake.py:3 fake.py:4 fake.py:5 fake.py:6 fake.py:7
#: fake.py:8 fake.py:9 fake.py:10 fake.py:11 fake.py:12 fake.py:13 fake.py:14
#: fake.py:15 fake.py:16 fake.py:17 fake.py:18 fake.py:19 fake.py:20 fake.py:21
#: fake.py:22 fake.py:23 fake.py:24 fake.py:25 fake.py:26 fake.py:27 fake.py:28
#: fake.py:29
msgid "pretty dam long message id, which must really be big to test this wrap behaviour, if not it won't work."
msgstr ""
""", buf.getvalue().lower())
buf = BytesIO()
pofile.write_po(buf, catalog, width=100, omit_header=True)
self.assertEqual(b"""\
#: fake.py:1 fake.py:2 fake.py:3 fake.py:4 fake.py:5 fake.py:6 fake.py:7 fake.py:8 fake.py:9 fake.py:10
#: fake.py:11 fake.py:12 fake.py:13 fake.py:14 fake.py:15 fake.py:16 fake.py:17 fake.py:18 fake.py:19
#: fake.py:20 fake.py:21 fake.py:22 fake.py:23 fake.py:24 fake.py:25 fake.py:26 fake.py:27 fake.py:28
#: fake.py:29
msgid ""
"pretty dam long message id, which must really be big to test this wrap behaviour, if not it won't"
" work."
msgstr ""
""", buf.getvalue().lower())
def test_with_context(self):
buf = BytesIO(b'''# Some string in the menu
#: main.py:1
msgctxt "Menu"
msgid "foo"
msgstr "Voh"
# Another string in the menu
#: main.py:2
msgctxt "Menu"
msgid "bar"
msgstr "Bahr"
''')
catalog = pofile.read_po(buf, ignore_obsolete=True)
self.assertEqual(2, len(catalog))
message = catalog.get('foo', context='Menu')
self.assertEqual('Menu', message.context)
message = catalog.get('bar', context='Menu')
self.assertEqual('Menu', message.context)
# And verify it pass through write_po
out_buf = BytesIO()
pofile.write_po(out_buf, catalog, omit_header=True)
assert out_buf.getvalue().strip() == buf.getvalue().strip(), \
out_buf.getvalue()
def test_write_po_file_with_specified_charset(self):
catalog = Catalog(charset='iso-8859-1')
catalog.add('foo', u'äöü', locations=[('main.py', 1)])
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=False)
po_file = buf.getvalue().strip()
assert b'"Content-Type: text/plain; charset=iso-8859-1\\n"' in po_file
assert u'msgstr "äöü"'.encode('iso-8859-1') in po_file
def test_duplicate_comments(self):
catalog = Catalog()
catalog.add(u'foo', auto_comments=['A comment'])
catalog.add(u'foo', auto_comments=['A comment'])
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=True)
self.assertEqual(b'''#. A comment
msgid "foo"
msgstr ""''', buf.getvalue().strip())
def test_join_locations(self):
catalog = Catalog()
catalog.add(u'foo', locations=[('main.py', 1)])
catalog.add(u'foo', locations=[('utils.py', 3)])
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=True)
self.assertEqual(b'''#: main.py:1 utils.py:3
msgid "foo"
msgstr ""''', buf.getvalue().strip())
def test_po_with_previous_msgid(self):
catalog = Catalog()
catalog.add(u'foo', u'Voh', locations=[('main.py', 1)],
previous_id=u'fo')
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=True, include_previous=True)
self.assertEqual(b'''#: main.py:1
#| msgid "fo"
msgid "foo"
msgstr "Voh"''', buf.getvalue().strip())
def test_po_with_obsolete_message_ignored(self):
catalog = Catalog()
catalog.add(u'foo', u'Voh', locations=[('main.py', 1)])
catalog.obsolete['bar'] = Message(u'bar', u'Bahr',
locations=[('utils.py', 3)],
user_comments=['User comment'])
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=True, ignore_obsolete=True)
self.assertEqual(b'''#: main.py:1
msgid "foo"
msgstr "Voh"''', buf.getvalue().strip())
def test_wrap_locations_with_hyphens(self):
catalog = Catalog()
catalog.add(u'foo', locations=[
('doupy/templates/base/navmenu.inc.html.py', 60)
])
catalog.add(u'foo', locations=[
('doupy/templates/job-offers/helpers.html', 22)
])
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=True)
self.assertEqual(b'''#: doupy/templates/base/navmenu.inc.html.py:60
#: doupy/templates/job-offers/helpers.html:22
msgid "foo"
msgstr ""''', buf.getvalue().strip())
def test_wrap_long_lines_with_long_word(self):
text = """Here's some text that
includesareallylongwordthatmightbutshouldnt throw us into an infinite loop
"""
catalog = Catalog()
catalog.add(text, locations=[('main.py', 1)])
buf = BytesIO()
pofile.write_po(buf, catalog, no_location=True, omit_header=True,
width=32)
self.assertEqual(b'''msgid ""
"Here's some text that\\n"
"includesareallylongwordthatmightbutshouldnt"
" throw us into an infinite "
"loop\\n"
msgstr ""''', buf.getvalue().strip())
def run(self):
mappings = self._get_mappings()
outfile = open(self.output_file, "wb")
try:
catalog = Catalog(
project=self.distribution.get_name(),
version=self.distribution.get_version(),
msgid_bugs_address=self.msgid_bugs_address,
copyright_holder=self.copyright_holder,
charset=self.charset,
)
for dirname, (method_map, options_map) in mappings.items():
def callback(filename, method, options):
if method == "ignore":
return
filepath = os.path.normpath(os.path.join(dirname, filename))
optstr = ""
if options:
optstr = " (%s)" % ", ".join(['%s="%s"' % (k, v) for k, v in options.items()])
log.info("extracting messages from %s%s", filepath, optstr)
extracted = extract_from_dir(
dirname,
method_map,
options_map,
keywords=self._keywords,
comment_tags=self._add_comments,
callback=callback,
strip_comment_tags=self.strip_comments,
)
for filename, lineno, message, comments, context in extracted:
filepath = os.path.normpath(os.path.join(dirname, filename))
catalog.add(message, None, [(filepath, lineno)], auto_comments=comments, context=context)
log.info("writing PO template file to %s" % self.output_file)
write_po(
outfile,
catalog,
width=self.width,
no_location=self.no_location,
omit_header=self.omit_header,
sort_output=self.sort_output,
sort_by_file=self.sort_by_file,
)
finally:
outfile.close()
def test_wrap_long_lines_in_header(self):
"""
Verify that long lines in the header comment are wrapped correctly.
"""
catalog = Catalog(project='AReallyReallyLongNameForAProject',
revision_date=datetime(2007, 4, 1))
buf = BytesIO()
pofile.write_po(buf, catalog)
self.assertEqual(b'''\
# Translations template for AReallyReallyLongNameForAProject.
# Copyright (C) 2007 ORGANIZATION
# This file is distributed under the same license as the
# AReallyReallyLongNameForAProject project.
# FIRST AUTHOR <EMAIL@ADDRESS>, 2007.
#
#, fuzzy''', b'\n'.join(buf.getvalue().splitlines()[:7]))
def test_pot_with_translator_comments(self):
catalog = Catalog()
catalog.add(u'foo', locations=[('main.py', 1)],
auto_comments=['Comment About `foo`'])
catalog.add(u'bar', locations=[('utils.py', 3)],
user_comments=['Comment About `bar` with',
'multiple lines.'])
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=True)
self.assertEqual(b'''#. Comment About `foo`
#: main.py:1
msgid "foo"
msgstr ""
# Comment About `bar` with
# multiple lines.
#: utils.py:3
msgid "bar"
msgstr ""''', buf.getvalue().strip())
def run(self):
log.info("creating catalog %r based on %r", self.output_file, self.input_file)
infile = open(self.input_file, "rb")
try:
# Although reading from the catalog template, read_po must be fed
# the locale in order to correctly calculate plurals
catalog = read_po(infile, locale=self.locale)
finally:
infile.close()
catalog.locale = self._locale
catalog.revision_date = datetime.now(LOCALTZ)
catalog.fuzzy = False
outfile = open(self.output_file, "wb")
try:
write_po(outfile, catalog, width=self.width)
finally:
outfile.close()
def test_wrap_long_lines(self):
text = """Here's some text where
white space and line breaks matter, and should
not be removed
"""
catalog = Catalog()
catalog.add(text, locations=[('main.py', 1)])
buf = BytesIO()
pofile.write_po(buf, catalog, no_location=True, omit_header=True,
width=42)
self.assertEqual(b'''msgid ""
"Here's some text where\\n"
"white space and line breaks matter, and"
" should\\n"
"\\n"
"not be removed\\n"
"\\n"
msgstr ""''', buf.getvalue().strip())
def test_sorted_po(self):
catalog = Catalog()
catalog.add(u'bar', locations=[('utils.py', 3)],
user_comments=['Comment About `bar` with',
'multiple lines.'])
catalog.add((u'foo', u'foos'), (u'Voh', u'Voeh'),
locations=[('main.py', 1)])
buf = BytesIO()
pofile.write_po(buf, catalog, sort_output=True)
value = buf.getvalue().strip()
assert b'''\
# Comment About `bar` with
# multiple lines.
#: utils.py:3
msgid "bar"
msgstr ""
#: main.py:1
msgid "foo"
msgid_plural "foos"
msgstr[0] "Voh"
msgstr[1] "Voeh"''' in value
assert value.find(b'msgid ""') < value.find(b'msgid "bar"') < value.find(b'msgid "foo"')
def update(self, argv):
"""Subcommand for updating existing message catalogs from a template.
:param argv: the command arguments
:since: version 0.9
"""
parser = OptionParser(usage=self.usage % ("update", ""), description=self.commands["update"])
parser.add_option("--domain", "-D", dest="domain", help="domain of PO file (default '%default')")
parser.add_option("--input-file", "-i", dest="input_file", metavar="FILE", help="name of the input file")
parser.add_option("--output-dir", "-d", dest="output_dir", metavar="DIR", help="path to output directory")
parser.add_option(
"--output-file",
"-o",
dest="output_file",
metavar="FILE",
help="name of the output file (default " "'<output_dir>/<locale>/LC_MESSAGES/" "<domain>.po')",
)
parser.add_option("--locale", "-l", dest="locale", metavar="LOCALE", help="locale of the translations catalog")
parser.add_option("-w", "--width", dest="width", type="int", help="set output line width (default 76)")
parser.add_option(
"--no-wrap",
dest="no_wrap",
action="store_true",
help="do not break long message lines, longer than " "the output line width, into several lines",
)
parser.add_option(
"--ignore-obsolete",
dest="ignore_obsolete",
action="store_true",
help="do not include obsolete messages in the output " "(default %default)",
)
parser.add_option(
"--no-fuzzy-matching",
"-N",
dest="no_fuzzy_matching",
action="store_true",
help="do not use fuzzy matching (default %default)",
)
parser.add_option(
"--previous",
dest="previous",
action="store_true",
help="keep previous msgids of translated messages " "(default %default)",
)
parser.set_defaults(domain="messages", ignore_obsolete=False, no_fuzzy_matching=False, previous=False)
options, args = parser.parse_args(argv)
if not options.input_file:
parser.error("you must specify the input file")
if not options.output_file and not options.output_dir:
parser.error("you must specify the output file or directory")
if options.output_file and not options.locale:
parser.error("you must specify the locale")
if options.no_fuzzy_matching and options.previous:
options.previous = False
po_files = []
if not options.output_file:
if options.locale:
po_files.append(
(
options.locale,
os.path.join(options.output_dir, options.locale, "LC_MESSAGES", options.domain + ".po"),
)
)
else:
for locale in os.listdir(options.output_dir):
po_file = os.path.join(options.output_dir, locale, "LC_MESSAGES", options.domain + ".po")
if os.path.exists(po_file):
po_files.append((locale, po_file))
else:
po_files.append((options.locale, options.output_file))
domain = options.domain
if not domain:
domain = os.path.splitext(os.path.basename(options.input_file))[0]
infile = open(options.input_file, "U")
try:
template = read_po(infile)
finally:
infile.close()
if not po_files:
parser.error("no message catalogs found")
if options.width and options.no_wrap:
parser.error("'--no-wrap' and '--width' are mutually exclusive.")
elif not options.width and not options.no_wrap:
options.width = 76
for locale, filename in po_files:
self.log.info("updating catalog %r based on %r", filename, options.input_file)
infile = open(filename, "rb")
try:
catalog = read_po(infile, locale=locale, domain=domain)
finally:
infile.close()
catalog.update(template, options.no_fuzzy_matching)
tmpname = os.path.join(os.path.dirname(filename), tempfile.gettempprefix() + os.path.basename(filename))
tmpfile = open(tmpname, "wb")
try:
try:
write_po(
tmpfile,
catalog,
ignore_obsolete=options.ignore_obsolete,
include_previous=options.previous,
width=options.width,
)
finally:
tmpfile.close()
except:
os.remove(tmpname)
raise
try:
os.rename(tmpname, filename)
except OSError:
# We're probably on Windows, which doesn't support atomic
# renames, at least not through Python
# If the error is in fact due to a permissions problem, that
# same error is going to be raised from one of the following
# operations
os.remove(filename)
shutil.copy(tmpname, filename)
os.remove(tmpname)
def extract(self, argv):
"""Subcommand for extracting messages from source files and generating
a POT file.
:param argv: the command arguments
"""
parser = OptionParser(usage=self.usage % ("extract", "dir1 <dir2> ..."), description=self.commands["extract"])
parser.add_option("--charset", dest="charset", help="charset to use in the output (default " '"%default")')
parser.add_option(
"-k",
"--keyword",
dest="keywords",
action="append",
help="keywords to look for in addition to the "
"defaults. You can specify multiple -k flags on "
"the command line.",
)
parser.add_option(
"--no-default-keywords",
dest="no_default_keywords",
action="store_true",
help="do not include the default keywords",
)
parser.add_option("--mapping", "-F", dest="mapping_file", help="path to the extraction mapping file")
parser.add_option(
"--no-location",
dest="no_location",
action="store_true",
help="do not include location comments with filename " "and line number",
)
parser.add_option(
"--omit-header", dest="omit_header", action="store_true", help='do not include msgid "" entry in header'
)
parser.add_option("-o", "--output", dest="output", help="path to the output POT file")
parser.add_option("-w", "--width", dest="width", type="int", help="set output line width (default 76)")
parser.add_option(
"--no-wrap",
dest="no_wrap",
action="store_true",
help="do not break long message lines, longer than " "the output line width, into several lines",
)
parser.add_option(
"--sort-output", dest="sort_output", action="store_true", help="generate sorted output (default False)"
)
parser.add_option(
"--sort-by-file",
dest="sort_by_file",
action="store_true",
help="sort output by file location (default False)",
)
parser.add_option(
"--msgid-bugs-address",
dest="msgid_bugs_address",
metavar="EMAIL@ADDRESS",
help="set report address for msgid",
)
parser.add_option("--copyright-holder", dest="copyright_holder", help="set copyright holder in output")
parser.add_option("--project", dest="project", help="set project name in output")
parser.add_option("--version", dest="version", help="set project version in output")
parser.add_option(
"--add-comments",
"-c",
dest="comment_tags",
metavar="TAG",
action="append",
help="place comment block with TAG (or those "
"preceding keyword lines) in output file. One "
"TAG per argument call",
)
parser.add_option(
"--strip-comment-tags",
"-s",
dest="strip_comment_tags",
action="store_true",
help="Strip the comment tags from the comments.",
)
parser.set_defaults(
charset="utf-8",
keywords=[],
no_default_keywords=False,
no_location=False,
omit_header=False,
width=None,
no_wrap=False,
sort_output=False,
sort_by_file=False,
comment_tags=[],
strip_comment_tags=False,
)
options, args = parser.parse_args(argv)
if not args:
parser.error("incorrect number of arguments")
keywords = DEFAULT_KEYWORDS.copy()
if options.no_default_keywords:
if not options.keywords:
parser.error("you must specify new keywords if you disable the " "default ones")
keywords = {}
if options.keywords:
keywords.update(parse_keywords(options.keywords))
if options.mapping_file:
fileobj = open(options.mapping_file, "U")
try:
method_map, options_map = parse_mapping(fileobj)
finally:
fileobj.close()
else:
method_map = DEFAULT_MAPPING
options_map = {}
if options.width and options.no_wrap:
parser.error("'--no-wrap' and '--width' are mutually exclusive.")
elif not options.width and not options.no_wrap:
options.width = 76
if options.sort_output and options.sort_by_file:
parser.error("'--sort-output' and '--sort-by-file' are mutually " "exclusive")
catalog = Catalog(
project=options.project,
version=options.version,
msgid_bugs_address=options.msgid_bugs_address,
copyright_holder=options.copyright_holder,
charset=options.charset,
)
for dirname in args:
if not os.path.isdir(dirname):
parser.error("%r is not a directory" % dirname)
def callback(filename, method, options):
if method == "ignore":
return
filepath = os.path.normpath(os.path.join(dirname, filename))
optstr = ""
if options:
optstr = " (%s)" % ", ".join(['%s="%s"' % (k, v) for k, v in options.items()])
self.log.info("extracting messages from %s%s", filepath, optstr)
extracted = extract_from_dir(
dirname,
method_map,
options_map,
keywords,
options.comment_tags,
callback=callback,
strip_comment_tags=options.strip_comment_tags,
)
for filename, lineno, message, comments, context in extracted:
filepath = os.path.normpath(os.path.join(dirname, filename))
catalog.add(message, None, [(filepath, lineno)], auto_comments=comments, context=context)
catalog_charset = catalog.charset
if options.output not in (None, "-"):
self.log.info("writing PO template file to %s" % options.output)
outfile = open(options.output, "wb")
close_output = True
else:
outfile = sys.stdout
# This is a bit of a hack on Python 3. stdout is a text stream so
# we need to find the underlying file when we write the PO. In
# later versions of Babel we want the write_po function to accept
# text or binary streams and automatically adjust the encoding.
if not PY2 and hasattr(outfile, "buffer"):
catalog.charset = outfile.encoding
outfile = outfile.buffer.raw
close_output = False
try:
write_po(
outfile,
catalog,
width=options.width,
no_location=options.no_location,
omit_header=options.omit_header,
sort_output=options.sort_output,
sort_by_file=options.sort_by_file,
)
finally:
if close_output:
outfile.close()
catalog.charset = catalog_charset
def run(self):
po_files = []
if not self.output_file:
if self.locale:
po_files.append(
(self.locale, os.path.join(self.output_dir, self.locale, "LC_MESSAGES", self.domain + ".po"))
)
else:
for locale in os.listdir(self.output_dir):
po_file = os.path.join(self.output_dir, locale, "LC_MESSAGES", self.domain + ".po")
if os.path.exists(po_file):
po_files.append((locale, po_file))
else:
po_files.append((self.locale, self.output_file))
domain = self.domain
if not domain:
domain = os.path.splitext(os.path.basename(self.input_file))[0]
infile = open(self.input_file, "rb")
try:
template = read_po(infile)
finally:
infile.close()
if not po_files:
raise DistutilsOptionError("no message catalogs found")
for locale, filename in po_files:
log.info("updating catalog %r based on %r", filename, self.input_file)
infile = open(filename, "rb")
try:
catalog = read_po(infile, locale=locale, domain=domain)
finally:
infile.close()
catalog.update(template, self.no_fuzzy_matching)
tmpname = os.path.join(os.path.dirname(filename), tempfile.gettempprefix() + os.path.basename(filename))
tmpfile = open(tmpname, "wb")
try:
try:
write_po(
tmpfile,
catalog,
ignore_obsolete=self.ignore_obsolete,
include_previous=self.previous,
width=self.width,
)
finally:
tmpfile.close()
except:
os.remove(tmpname)
raise
try:
os.rename(tmpname, filename)
except OSError:
# We're probably on Windows, which doesn't support atomic
# renames, at least not through Python
# If the error is in fact due to a permissions problem, that
# same error is going to be raised from one of the following
# operations
os.remove(filename)
shutil.copy(tmpname, filename)
os.remove(tmpname)
def init(self, argv):
"""Subcommand for creating new message catalogs from a template.
:param argv: the command arguments
"""
parser = OptionParser(usage=self.usage % ("init", ""), description=self.commands["init"])
parser.add_option("--domain", "-D", dest="domain", help="domain of PO file (default '%default')")
parser.add_option("--input-file", "-i", dest="input_file", metavar="FILE", help="name of the input file")
parser.add_option("--output-dir", "-d", dest="output_dir", metavar="DIR", help="path to output directory")
parser.add_option(
"--output-file",
"-o",
dest="output_file",
metavar="FILE",
help="name of the output file (default " "'<output_dir>/<locale>/LC_MESSAGES/" "<domain>.po')",
)
parser.add_option(
"--locale", "-l", dest="locale", metavar="LOCALE", help="locale for the new localized catalog"
)
parser.add_option("-w", "--width", dest="width", type="int", help="set output line width (default 76)")
parser.add_option(
"--no-wrap",
dest="no_wrap",
action="store_true",
help="do not break long message lines, longer than " "the output line width, into several lines",
)
parser.set_defaults(domain="messages")
options, args = parser.parse_args(argv)
if not options.locale:
parser.error("you must provide a locale for the new catalog")
try:
locale = Locale.parse(options.locale)
except UnknownLocaleError as e:
parser.error(e)
if not options.input_file:
parser.error("you must specify the input file")
if not options.output_file and not options.output_dir:
parser.error("you must specify the output file or directory")
if not options.output_file:
options.output_file = os.path.join(
options.output_dir, options.locale, "LC_MESSAGES", options.domain + ".po"
)
if not os.path.exists(os.path.dirname(options.output_file)):
os.makedirs(os.path.dirname(options.output_file))
if options.width and options.no_wrap:
parser.error("'--no-wrap' and '--width' are mutually exclusive.")
elif not options.width and not options.no_wrap:
options.width = 76
infile = open(options.input_file, "r")
try:
# Although reading from the catalog template, read_po must be fed
# the locale in order to correctly calculate plurals
catalog = read_po(infile, locale=options.locale)
finally:
infile.close()
catalog.locale = locale
catalog.revision_date = datetime.now(LOCALTZ)
self.log.info("creating catalog %r based on %r", options.output_file, options.input_file)
outfile = open(options.output_file, "wb")
try:
write_po(outfile, catalog, width=options.width)
finally:
outfile.close()
