def test_no_wrap_and_width_behaviour_on_comments(self):
catalog = Catalog()
catalog.add("Pretty dam long message id, which must really be big "
"to test this wrap behaviour, if not it won't work.",
locations=[("fake.py", n) for n in range(1, 30)])
buf = BytesIO()
pofile.write_po(buf, catalog, width=None, omit_header=True)
self.assertEqual(b"""\
#: fake.py:1 fake.py:2 fake.py:3 fake.py:4 fake.py:5 fake.py:6 fake.py:7
#: fake.py:8 fake.py:9 fake.py:10 fake.py:11 fake.py:12 fake.py:13 fake.py:14
#: fake.py:15 fake.py:16 fake.py:17 fake.py:18 fake.py:19 fake.py:20 fake.py:21
#: fake.py:22 fake.py:23 fake.py:24 fake.py:25 fake.py:26 fake.py:27 fake.py:28
#: fake.py:29
msgid "pretty dam long message id, which must really be big to test this wrap behaviour, if not it won't work."
msgstr ""
""", buf.getvalue().lower())
buf = BytesIO()
pofile.write_po(buf, catalog, width=100, omit_header=True)
self.assertEqual(b"""\
#: fake.py:1 fake.py:2 fake.py:3 fake.py:4 fake.py:5 fake.py:6 fake.py:7 fake.py:8 fake.py:9 fake.py:10
#: fake.py:11 fake.py:12 fake.py:13 fake.py:14 fake.py:15 fake.py:16 fake.py:17 fake.py:18 fake.py:19
#: fake.py:20 fake.py:21 fake.py:22 fake.py:23 fake.py:24 fake.py:25 fake.py:26 fake.py:27 fake.py:28
#: fake.py:29
msgid ""
"pretty dam long message id, which must really be big to test this wrap behaviour, if not it won't"
" work."
msgstr ""
""", buf.getvalue().lower())
def test_po_with_multiline_obsolete_message(self):
catalog = Catalog()
catalog.add(u'foo', u'Voh', locations=[('main.py', 1)])
msgid = r"""Here's a message that covers
multiple lines, and should still be handled
correctly.
"""
msgstr = r"""Here's a message that covers
multiple lines, and should still be handled
correctly.
"""
catalog.obsolete[msgid] = Message(msgid, msgstr,
locations=[('utils.py', 3)])
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=True)
self.assertEqual(b'''#: main.py:1
msgid "foo"
msgstr "Voh"
#~ msgid ""
#~ "Here's a message that covers\\n"
#~ "multiple lines, and should still be handled\\n"
#~ "correctly.\\n"
#~ msgstr ""
#~ "Here's a message that covers\\n"
#~ "multiple lines, and should still be handled\\n"
#~ "correctly.\\n"''', buf.getvalue().strip())
def test_write_po_file_with_specified_charset(self):
catalog = Catalog(charset='iso-8859-1')
catalog.add('foo', u'äöü', locations=[('main.py', 1)])
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=False)
po_file = buf.getvalue().strip()
assert b'"Content-Type: text/plain; charset=iso-8859-1\\n"' in po_file
assert u'msgstr "äöü"'.encode('iso-8859-1') in po_file
def test_duplicate_comments(self):
catalog = Catalog()
catalog.add(u'foo', auto_comments=['A comment'])
catalog.add(u'foo', auto_comments=['A comment'])
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=True)
self.assertEqual(b'''#. A comment
msgid "foo"
msgstr ""''', buf.getvalue().strip())
def test_join_locations(self):
catalog = Catalog()
catalog.add(u'foo', locations=[('main.py', 1)])
catalog.add(u'foo', locations=[('utils.py', 3)])
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=True)
self.assertEqual(b'''#: main.py:1 utils.py:3
msgid "foo"
msgstr ""''', buf.getvalue().strip())
def test_po_with_previous_msgid(self):
catalog = Catalog()
catalog.add(u'foo', u'Voh', locations=[('main.py', 1)],
previous_id=u'fo')
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=True, include_previous=True)
self.assertEqual(b'''#: main.py:1
#| msgid "fo"
msgid "foo"
msgstr "Voh"''', buf.getvalue().strip())
def test_po_with_obsolete_message_ignored(self):
catalog = Catalog()
catalog.add(u'foo', u'Voh', locations=[('main.py', 1)])
catalog.obsolete['bar'] = Message(u'bar', u'Bahr',
locations=[('utils.py', 3)],
user_comments=['User comment'])
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=True, ignore_obsolete=True)
self.assertEqual(b'''#: main.py:1
msgid "foo"
msgstr "Voh"''', buf.getvalue().strip())
def test_wrap_locations_with_hyphens(self):
catalog = Catalog()
catalog.add(u'foo', locations=[
('doupy/templates/base/navmenu.inc.html.py', 60)
])
catalog.add(u'foo', locations=[
('doupy/templates/job-offers/helpers.html', 22)
])
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=True)
self.assertEqual(b'''#: doupy/templates/base/navmenu.inc.html.py:60
#: doupy/templates/job-offers/helpers.html:22
msgid "foo"
msgstr ""''', buf.getvalue().strip())
def test_wrap_long_lines_with_long_word(self):
text = """Here's some text that
includesareallylongwordthatmightbutshouldnt throw us into an infinite loop
"""
catalog = Catalog()
catalog.add(text, locations=[('main.py', 1)])
buf = BytesIO()
pofile.write_po(buf, catalog, no_location=True, omit_header=True,
width=32)
self.assertEqual(b'''msgid ""
"Here's some text that\\n"
"includesareallylongwordthatmightbutshouldnt"
" throw us into an infinite "
"loop\\n"
msgstr ""''', buf.getvalue().strip())
def run(self):
mappings = self._get_mappings()
outfile = open(self.output_file, "wb")
try:
catalog = Catalog(
project=self.distribution.get_name(),
version=self.distribution.get_version(),
msgid_bugs_address=self.msgid_bugs_address,
copyright_holder=self.copyright_holder,
charset=self.charset,
)
for dirname, (method_map, options_map) in mappings.items():
def callback(filename, method, options):
if method == "ignore":
return
filepath = os.path.normpath(os.path.join(dirname, filename))
optstr = ""
if options:
optstr = " (%s)" % ", ".join(['%s="%s"' % (k, v) for k, v in options.items()])
log.info("extracting messages from %s%s", filepath, optstr)
extracted = extract_from_dir(
dirname,
method_map,
options_map,
keywords=self._keywords,
comment_tags=self._add_comments,
callback=callback,
strip_comment_tags=self.strip_comments,
)
for filename, lineno, message, comments, context in extracted:
filepath = os.path.normpath(os.path.join(dirname, filename))
catalog.add(message, None, [(filepath, lineno)], auto_comments=comments, context=context)
log.info("writing PO template file to %s" % self.output_file)
write_po(
outfile,
catalog,
width=self.width,
no_location=self.no_location,
omit_header=self.omit_header,
sort_output=self.sort_output,
sort_by_file=self.sort_by_file,
)
finally:
outfile.close()
def test_pot_with_translator_comments(self):
catalog = Catalog()
catalog.add(u'foo', locations=[('main.py', 1)],
auto_comments=['Comment About `foo`'])
catalog.add(u'bar', locations=[('utils.py', 3)],
user_comments=['Comment About `bar` with',
'multiple lines.'])
buf = BytesIO()
pofile.write_po(buf, catalog, omit_header=True)
self.assertEqual(b'''#. Comment About `foo`
#: main.py:1
msgid "foo"
msgstr ""
# Comment About `bar` with
# multiple lines.
#: utils.py:3
msgid "bar"
msgstr ""''', buf.getvalue().strip())
def test_wrap_long_lines(self):
text = """Here's some text where
white space and line breaks matter, and should
not be removed
"""
catalog = Catalog()
catalog.add(text, locations=[('main.py', 1)])
buf = BytesIO()
pofile.write_po(buf, catalog, no_location=True, omit_header=True,
width=42)
self.assertEqual(b'''msgid ""
"Here's some text where\\n"
"white space and line breaks matter, and"
" should\\n"
"\\n"
"not be removed\\n"
"\\n"
msgstr ""''', buf.getvalue().strip())
def test_sorted_po(self):
catalog = Catalog()
catalog.add(u'bar', locations=[('utils.py', 3)],
user_comments=['Comment About `bar` with',
'multiple lines.'])
catalog.add((u'foo', u'foos'), (u'Voh', u'Voeh'),
locations=[('main.py', 1)])
buf = BytesIO()
pofile.write_po(buf, catalog, sort_output=True)
value = buf.getvalue().strip()
assert b'''\
# Comment About `bar` with
# multiple lines.
#: utils.py:3
msgid "bar"
msgstr ""
#: main.py:1
msgid "foo"
msgid_plural "foos"
msgstr[0] "Voh"
msgstr[1] "Voeh"''' in value
assert value.find(b'msgid ""') < value.find(b'msgid "bar"') < value.find(b'msgid "foo"')
def read_mo(fileobj):
"""Read a binary MO file from the given file-like object and return a
corresponding `Catalog` object.
:param fileobj: the file-like object to read the MO file from
:note: The implementation of this function is heavily based on the
``GNUTranslations._parse`` method of the ``gettext`` module in the
standard library.
"""
catalog = Catalog()
headers = {}
filename = getattr(fileobj, 'name', '')
buf = fileobj.read()
buflen = len(buf)
unpack = struct.unpack
# Parse the .mo file header, which consists of 5 little endian 32
# bit words.
magic = unpack('<I', buf[:4])[0] # Are we big endian or little endian?
if magic == LE_MAGIC:
version, msgcount, origidx, transidx = unpack('<4I', buf[4:20])
ii = '<II'
elif magic == BE_MAGIC:
version, msgcount, origidx, transidx = unpack('>4I', buf[4:20])
ii = '>II'
else:
raise IOError(0, 'Bad magic number', filename)
# Now put all messages from the .mo file buffer into the catalog
# dictionary
for i in range_type(0, msgcount):
mlen, moff = unpack(ii, buf[origidx:origidx + 8])
mend = moff + mlen
tlen, toff = unpack(ii, buf[transidx:transidx + 8])
tend = toff + tlen
if mend < buflen and tend < buflen:
msg = buf[moff:mend]
tmsg = buf[toff:tend]
else:
raise IOError(0, 'File is corrupt', filename)
# See if we're looking at GNU .mo conventions for metadata
if mlen == 0:
# Catalog description
lastkey = key = None
for item in tmsg.splitlines():
item = item.strip()
if not item:
continue
if b':' in item:
key, value = item.split(b':', 1)
lastkey = key = key.strip().lower()
headers[key] = value.strip()
elif lastkey:
headers[lastkey] += b'\n' + item
if b'\x04' in msg: # context
ctxt, msg = msg.split(b'\x04')
else:
ctxt = None
if b'\x00' in msg: # plural forms
msg = msg.split(b'\x00')
tmsg = tmsg.split(b'\x00')
if catalog.charset:
msg = [x.decode(catalog.charset) for x in msg]
tmsg = [x.decode(catalog.charset) for x in tmsg]
else:
if catalog.charset:
msg = msg.decode(catalog.charset)
tmsg = tmsg.decode(catalog.charset)
catalog[msg] = Message(msg, tmsg, context=ctxt)
# advance to next entry in the seek tables
origidx += 8
transidx += 8
catalog.mime_headers = headers.items()
return catalog
def extract(self, argv):
"""Subcommand for extracting messages from source files and generating
a POT file.
:param argv: the command arguments
"""
parser = OptionParser(usage=self.usage % ("extract", "dir1 <dir2> ..."), description=self.commands["extract"])
parser.add_option("--charset", dest="charset", help="charset to use in the output (default " '"%default")')
parser.add_option(
"-k",
"--keyword",
dest="keywords",
action="append",
help="keywords to look for in addition to the "
"defaults. You can specify multiple -k flags on "
"the command line.",
)
parser.add_option(
"--no-default-keywords",
dest="no_default_keywords",
action="store_true",
help="do not include the default keywords",
)
parser.add_option("--mapping", "-F", dest="mapping_file", help="path to the extraction mapping file")
parser.add_option(
"--no-location",
dest="no_location",
action="store_true",
help="do not include location comments with filename " "and line number",
)
parser.add_option(
"--omit-header", dest="omit_header", action="store_true", help='do not include msgid "" entry in header'
)
parser.add_option("-o", "--output", dest="output", help="path to the output POT file")
parser.add_option("-w", "--width", dest="width", type="int", help="set output line width (default 76)")
parser.add_option(
"--no-wrap",
dest="no_wrap",
action="store_true",
help="do not break long message lines, longer than " "the output line width, into several lines",
)
parser.add_option(
"--sort-output", dest="sort_output", action="store_true", help="generate sorted output (default False)"
)
parser.add_option(
"--sort-by-file",
dest="sort_by_file",
action="store_true",
help="sort output by file location (default False)",
)
parser.add_option(
"--msgid-bugs-address",
dest="msgid_bugs_address",
metavar="EMAIL@ADDRESS",
help="set report address for msgid",
)
parser.add_option("--copyright-holder", dest="copyright_holder", help="set copyright holder in output")
parser.add_option("--project", dest="project", help="set project name in output")
parser.add_option("--version", dest="version", help="set project version in output")
parser.add_option(
"--add-comments",
"-c",
dest="comment_tags",
metavar="TAG",
action="append",
help="place comment block with TAG (or those "
"preceding keyword lines) in output file. One "
"TAG per argument call",
)
parser.add_option(
"--strip-comment-tags",
"-s",
dest="strip_comment_tags",
action="store_true",
help="Strip the comment tags from the comments.",
)
parser.set_defaults(
charset="utf-8",
keywords=[],
no_default_keywords=False,
no_location=False,
omit_header=False,
width=None,
no_wrap=False,
sort_output=False,
sort_by_file=False,
comment_tags=[],
strip_comment_tags=False,
)
options, args = parser.parse_args(argv)
if not args:
parser.error("incorrect number of arguments")
keywords = DEFAULT_KEYWORDS.copy()
if options.no_default_keywords:
if not options.keywords:
parser.error("you must specify new keywords if you disable the " "default ones")
keywords = {}
if options.keywords:
keywords.update(parse_keywords(options.keywords))
if options.mapping_file:
fileobj = open(options.mapping_file, "U")
try:
method_map, options_map = parse_mapping(fileobj)
finally:
fileobj.close()
else:
method_map = DEFAULT_MAPPING
options_map = {}
if options.width and options.no_wrap:
parser.error("'--no-wrap' and '--width' are mutually exclusive.")
elif not options.width and not options.no_wrap:
options.width = 76
if options.sort_output and options.sort_by_file:
parser.error("'--sort-output' and '--sort-by-file' are mutually " "exclusive")
catalog = Catalog(
project=options.project,
version=options.version,
msgid_bugs_address=options.msgid_bugs_address,
copyright_holder=options.copyright_holder,
charset=options.charset,
)
for dirname in args:
if not os.path.isdir(dirname):
parser.error("%r is not a directory" % dirname)
def callback(filename, method, options):
if method == "ignore":
return
filepath = os.path.normpath(os.path.join(dirname, filename))
optstr = ""
if options:
optstr = " (%s)" % ", ".join(['%s="%s"' % (k, v) for k, v in options.items()])
self.log.info("extracting messages from %s%s", filepath, optstr)
extracted = extract_from_dir(
dirname,
method_map,
options_map,
keywords,
options.comment_tags,
callback=callback,
strip_comment_tags=options.strip_comment_tags,
)
for filename, lineno, message, comments, context in extracted:
filepath = os.path.normpath(os.path.join(dirname, filename))
catalog.add(message, None, [(filepath, lineno)], auto_comments=comments, context=context)
catalog_charset = catalog.charset
if options.output not in (None, "-"):
self.log.info("writing PO template file to %s" % options.output)
outfile = open(options.output, "wb")
close_output = True
else:
outfile = sys.stdout
# This is a bit of a hack on Python 3. stdout is a text stream so
# we need to find the underlying file when we write the PO. In
# later versions of Babel we want the write_po function to accept
# text or binary streams and automatically adjust the encoding.
if not PY2 and hasattr(outfile, "buffer"):
catalog.charset = outfile.encoding
outfile = outfile.buffer.raw
close_output = False
try:
write_po(
outfile,
catalog,
width=options.width,
no_location=options.no_location,
omit_header=options.omit_header,
sort_output=options.sort_output,
sort_by_file=options.sort_by_file,
)
finally:
if close_output:
outfile.close()
catalog.charset = catalog_charset