def parse(self, translation_import_queue_entry): """See `ITranslationFormatImporter`.""" self.basepath = translation_import_queue_entry.path self.productseries = translation_import_queue_entry.productseries self.distroseries = translation_import_queue_entry.distroseries self.sourcepackagename = translation_import_queue_entry.sourcepackagename self.by_maintainer = translation_import_queue_entry.by_maintainer librarian_client = getUtility(ILibrarianClient) self.content = librarian_client.getFileByAlias(translation_import_queue_entry.content.id) pofile = translation_import_queue_entry.pofile if pofile is None: pluralformula = None else: pluralformula = pofile.language.pluralexpression parser = POParser(pluralformula) return parser.parse(self.content.read())
def parse(self, translation_import_queue_entry): """See `ITranslationFormatImporter`.""" self.basepath = translation_import_queue_entry.path self.productseries = translation_import_queue_entry.productseries self.distroseries = translation_import_queue_entry.distroseries self.sourcepackagename = ( translation_import_queue_entry.sourcepackagename) self.by_maintainer = translation_import_queue_entry.by_maintainer librarian_client = getUtility(ILibrarianClient) self.content = librarian_client.getFileByAlias( translation_import_queue_entry.content.id) pofile = translation_import_queue_entry.pofile if pofile is None: pluralformula = None else: pluralformula = pofile.language.pluralexpression parser = POParser(pluralformula) return parser.parse(self.content.read())
def _makeTranslationFileData(self, is_template=False): """Produce a TranslationFileData with one message: "foo".""" file_data = POParser().parse(dedent(""" msgid "" msgstr "" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=UTF-8\\n" msgid "foo" msgstr "bar" """)) file_data.is_template = is_template file_data.language_code = 'my' file_data.translation_domain = 'main' if is_template: file_data.path = file_data.translation_domain + '.pot' else: file_data.path = file_data.language_code + '.po' return file_data
def validate_po(filename, content): """Validate a gettext PO or POT file.""" POParser().parse(content)
def setUp(self): TestCaseWithFactory.setUp(self) self.parser = POParser() self.translation_exporter = GettextPOExporter()
class GettextPOExporterTestCase(TestCaseWithFactory): """Class test for gettext's .po file exports""" layer = LaunchpadZopelessLayer def setUp(self): TestCaseWithFactory.setUp(self) self.parser = POParser() self.translation_exporter = GettextPOExporter() def _compareImportAndExport(self, import_file, export_file): """Compare imported file and the export we got from it. :param import_file: buffer with the source file content. :param export_file: buffer with the output file content. """ import_lines = [line for line in import_file.split('\n')] # Remove X-Launchpad-Export-Date line to prevent time bombs in tests. export_lines = [ line for line in export_file.split('\n') if (not line.startswith('"X-Launchpad-Export-Date:') and not line.startswith('"X-Generator: Launchpad'))] line_pairs = zip(export_lines, import_lines) debug_diff = test_diff(import_lines, export_lines) for export_line, import_line in line_pairs: self.assertEqual( export_line, import_line, "Output doesn't match:\n\n %s" % debug_diff) self.assertEqual( len(export_lines), len(import_lines), "Output has excess lines:\n\n %s" % debug_diff) def testInterface(self): """Check whether the object follows the interface.""" self.assertTrue( verifyObject(ITranslationFormatExporter, self.translation_exporter), "GettextPOExporter doesn't follow the interface") def testSupportedFormats(self): """Check that the exporter reports the correct formats.""" self.assertEqual( self.translation_exporter.format, TranslationFileFormat.PO, "Expected GettextPOExporter to provide PO format " "but got %r instead." % self.translation_exporter.format) self.assertEqual( self.translation_exporter.supported_source_formats, [TranslationFileFormat.PO, TranslationFileFormat.KDEPO], "Expected GettextPOExporter to support PO and KDEPO source " "formats but got %r instead." % ( self.translation_exporter.supported_source_formats)) def testGeneralExport(self): """Check different kind of messages export.""" pofile_cy = dedent(''' msgid "" msgstr "" "Project-Id-Version: foo\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: Kubla Kahn <*****@*****.**>\\n" "Language-Team: LANGUAGE <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=UTF-8\\n" "Content-Transfer-Encoding: 8bit\\n" "Plural-Forms: nplurals=4; plural=n==1 ? 0 : n==2 ? 1 : (n != 8 || n != 11) ? " "2 : 3;\\n" msgid "foo" msgid_plural "foos" msgstr[0] "cy-F001" msgstr[1] "cy-F002" msgstr[2] "" msgstr[3] "" #, fuzzy #| msgid "zog" msgid "zig" msgstr "zag" #, c-format msgid "zip" msgstr "zap" # tove #. borogove #: rath msgid "zog" msgstr "zug" #~ msgid "zot" #~ msgstr "zat" ''') cy_translation_file = self.parser.parse(pofile_cy) cy_translation_file.is_template = False cy_translation_file.language_code = 'cy' cy_translation_file.path = 'po/cy.po' cy_translation_file.translation_domain = 'testing' storage = ExportFileStorage() self.translation_exporter.exportTranslationFile( cy_translation_file, storage) self._compareImportAndExport( pofile_cy.strip(), storage.export().read().strip()) def testObsoleteExport(self): """Check how obsoleted messages are exported.""" pofile_eo = dedent(''' msgid "" msgstr "" "Project-Id-Version: Kumquats 1.0\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: L.L. Zamenhoff <*****@*****.**>\\n" "Language-Team: Esperanto <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=UTF-8\\n" "Content-Transfer-Encoding: 8bit\\n" # Foo bar. #, c-format #: src/foo.c #| msgid "zog" msgid "zig" msgstr "zag" ''') pofile_eo_obsolete = dedent(''' msgid "" msgstr "" "Project-Id-Version: Kumquats 1.0\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: L.L. Zamenhoff <*****@*****.**>\\n" "Language-Team: Esperanto <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=UTF-8\\n" "Content-Transfer-Encoding: 8bit\\n" # Foo bar. #, c-format #~| msgid "zog" #~ msgid "zig" #~ msgstr "zag" ''') eo_translation_file = self.parser.parse(pofile_eo) eo_translation_file.is_template = False eo_translation_file.language_code = 'eo' eo_translation_file.path = 'po/eo.po' eo_translation_file.translation_domain = 'testing' eo_translation_file.messages[0].is_obsolete = True storage = ExportFileStorage() self.translation_exporter.exportTranslationFile( eo_translation_file, storage) self._compareImportAndExport( pofile_eo_obsolete.strip(), storage.export().read().strip()) def testEncodingExport(self): """Test that PO headers specifying character sets are respected.""" def compare(self, pofile): "Compare the original text with the exported one.""" # This is the word 'Japanese' in Japanese, in Unicode. nihongo_unicode = u'\u65e5\u672c\u8a9e' translation_file = self.parser.parse(pofile) translation_file.is_template = False translation_file.language_code = 'ja' translation_file.path = 'po/ja.po' translation_file.translation_domain = 'testing' # We are sure that 'Japanese' is correctly stored as Unicode so # we are sure the exporter does its job instead of just export # what was imported. self.assertEqual( translation_file.messages[0].translations, [nihongo_unicode]) storage = ExportFileStorage() self.translation_exporter.exportTranslationFile( translation_file, storage) self._compareImportAndExport( pofile.strip(), storage.export().read().strip()) # File representing the same PO file three times. Each is identical # except for the charset declaration in the header. pofiles = [ dedent(''' msgid "" msgstr "" "Project-Id-Version: foo\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: Kubla Kahn <*****@*****.**>\\n" "Language-Team: LANGUAGE <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=UTF-8\\n" "Content-Transfer-Encoding: 8bit\\n" msgid "Japanese" msgstr "\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e" '''), dedent(''' msgid "" msgstr "" "Project-Id-Version: foo\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: Kubla Kahn <*****@*****.**>\\n" "Language-Team: LANGUAGE <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=Shift-JIS\\n" "Content-Transfer-Encoding: 8bit\\n" msgid "Japanese" msgstr "\x93\xfa\x96\x7b\x8c\xea" '''), dedent(''' msgid "" msgstr "" "Project-Id-Version: foo\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: Kubla Kahn <*****@*****.**>\\n" "Language-Team: LANGUAGE <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=EUC-JP\\n" "Content-Transfer-Encoding: 8bit\\n" msgid "Japanese" msgstr "\xc6\xfc\xcb\xdc\xb8\xec" '''), ] for pofile in pofiles: compare(self, pofile) def _testBrokenEncoding(self, pofile_content): translation_file = self.parser.parse( pofile_content % {'charset': 'ISO-8859-15', 'special': '\xe1'}) translation_file.is_template = False translation_file.language_code = 'es' translation_file.path = 'po/es.po' translation_file.translation_domain = 'testing' # Force the export as ASCII, it will not be possible because # translation is not available in that encoding and thus, we should # get an export in UTF-8. translation_file.header.charset = 'ASCII' storage = ExportFileStorage() self.translation_exporter.exportTranslationFile( translation_file, storage) self._compareImportAndExport( pofile_content.strip() % { 'charset': 'UTF-8', 'special': '\xc3\xa1'}, storage.export().read().strip()) def testBrokenEncodingExport(self): """Test what happens when the content and the encoding don't agree. If a pofile fails to encode using the character set specified in the header, the header should be changed to specify to UTF-8 and the pofile exported accordingly. """ pofile_content = dedent(''' msgid "" msgstr "" "Project-Id-Version: foo\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: Kubla Kahn <*****@*****.**>\\n" "Language-Team: LANGUAGE <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=%(charset)s\\n" "Content-Transfer-Encoding: 8bit\\n" msgid "a" msgstr "%(special)s" ''') self._testBrokenEncoding(pofile_content) def testBrokenEncodingHeader(self): """A header field might require a different encoding, too. This usually happens if the Last-Translator name contains non-ascii characters. If a pofile fails to encode using the character set specified in the header, the header should be changed to specify to UTF-8 and the pofile exported accordingly. """ pofile_content = dedent(''' msgid "" msgstr "" "Project-Id-Version: foo\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: Kubla K%(special)shn <*****@*****.**>\\n" "Language-Team: LANGUAGE <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=%(charset)s\\n" "Content-Transfer-Encoding: 8bit\\n" msgid "a" msgstr "b" ''') self._testBrokenEncoding(pofile_content) def testIncompletePluralMessage(self): """Test export correctness for partial plural messages.""" pofile = dedent(''' msgid "" msgstr "" "Project-Id-Version: foo\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: Kubla Kahn <*****@*****.**>\\n" "Language-Team: LANGUAGE <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=UTF-8\\n" "Content-Transfer-Encoding: 8bit\\n" "Plural-Forms: nplurals=2; plural=(n != 1);\\n" msgid "1 dead horse" msgid_plural "%%d dead horses" msgstr[0] "ning\xc3\xban caballo muerto" %s''') translation_file = self.parser.parse(pofile % ('')) translation_file.is_template = False translation_file.language_code = 'es' translation_file.path = 'po/es.po' translation_file.translation_domain = 'testing' storage = ExportFileStorage() self.translation_exporter.exportTranslationFile( translation_file, storage) self._compareImportAndExport( pofile.strip() % 'msgstr[1] ""', storage.export().read().strip()) def testClashingSingularMsgIds(self): # We don't accept it in gettext imports directly, since it's not # valid gettext, but it's possible for our database to hold # messages that differ only in msgid_plural. In gettext those # would be considered equal, so we can't export them. Only the # first of the two messages is exported. template = self.factory.makePOTemplate() self.factory.makePOTMsgSet( template, singular='%d foo', plural='%d foos', sequence=1) self.factory.makePOTMsgSet( template, singular='%d foo', plural='%d foox', sequence=2) exported_file = template.export() # The "foos" (as opposed to "foox") tells us that the exporter # has picked the first message for export. expected_output = dedent(""" msgid "%d foo" msgid_plural "%d foos" msgstr[0] "" msgstr[1] "" """).strip() body = exported_file.split('\n\n', 1)[1].strip() self.assertEqual(body, expected_output) def testObsoleteMessageYieldsToNonObsoleteClashingOne(self): # When an obsolete message and a non-obsolete message in the # same POFile have identical identifying information except # msgid_plural (which Launchpad considers part of the message's # identifying information but gettext does not), only the # non-obsolete one is exported. template = self.factory.makePOTemplate() obsolete_message = self.factory.makePOTMsgSet( template, singular='%d goo', plural='%d goos', sequence=0) current_message = self.factory.makePOTMsgSet( template, singular='%d goo', plural='%d gooim', sequence=1) pofile = self.factory.makePOFile( potemplate=template, language_code='nl') self.factory.makeCurrentTranslationMessage( pofile=pofile, potmsgset=obsolete_message, translations=['%d splut', '%d splutjes']) self.factory.makeCurrentTranslationMessage( pofile=pofile, potmsgset=current_message, translations=['%d gargl', '%d garglii']) exported_file = pofile.export() # The "gooim" (as opposed to "goos") tells us that the exporter # has picked the non-obsolete message for export. The "gargl" # and "garglii" tell us we're not just getting the msgid from # the non-obsolete message, but the translations as well. expected_output = dedent(""" msgid "%d goo" msgid_plural "%d gooim" msgstr[0] "%d gargl" msgstr[1] "%d garglii" """).strip() body = exported_file.split('\n\n', 1)[1].strip() self.assertEqual(expected_output, body) def test_strip_last_newline(self): # `strip_last_newline` strips only the last newline. self.assertEqual('text\n', strip_last_newline('text\n\n')) self.assertEqual('text\nx', strip_last_newline('text\nx')) self.assertEqual('text', strip_last_newline('text')) # It supports '\r' as well (Mac-style). self.assertEqual('text', strip_last_newline('text\r')) # And DOS-style '\r\n'. self.assertEqual('text', strip_last_newline('text\r\n')) # With weird combinations, it strips only the last # newline-indicating character. self.assertEqual('text\n', strip_last_newline('text\n\r')) def test_comments_text_representation_multiline(self): # Comments with newlines should be correctly exported. data = TranslationMessageData() data.comment = "Line One\nLine Two" self.assertEqual("#Line One\n#Line Two", comments_text_representation(data)) # It works the same when there's a final newline as well. data.comment = "Line One\nLine Two\n" self.assertEqual("#Line One\n#Line Two", comments_text_representation(data)) # And similar processing happens for source comments. data = TranslationMessageData() data.source_comment = "Line One\nLine Two" self.assertEqual("#. Line One\n#. Line Two", comments_text_representation(data)) # It works the same when there's a final newline as well. data.source_comment = "Line One\nLine Two\n" self.assertEqual("#. Line One\n#. Line Two", comments_text_representation(data)) def test_export_handles_empty_files(self): # Exporting an empty gettext file does not break the exporter. # The output does contain one message: the header. output = self.factory.makePOFile('nl').export() self.assertTrue(output.startswith('# Dutch translation for ')) self.assertEqual(1, output.count('msgid'))
class GettextPOExporterTestCase(TestCaseWithFactory): """Class test for gettext's .po file exports""" layer = LaunchpadZopelessLayer def setUp(self): TestCaseWithFactory.setUp(self) self.parser = POParser() self.translation_exporter = GettextPOExporter() def _compareImportAndExport(self, import_file, export_file): """Compare imported file and the export we got from it. :param import_file: buffer with the source file content. :param export_file: buffer with the output file content. """ import_lines = [line for line in import_file.split("\n")] # Remove X-Launchpad-Export-Date line to prevent time bombs in tests. export_lines = [ line for line in export_file.split("\n") if (not line.startswith('"X-Launchpad-Export-Date:') and not line.startswith('"X-Generator: Launchpad')) ] line_pairs = zip(export_lines, import_lines) debug_diff = test_diff(import_lines, export_lines) for export_line, import_line in line_pairs: self.assertEqual(export_line, import_line, "Output doesn't match:\n\n %s" % debug_diff) self.assertEqual(len(export_lines), len(import_lines), "Output has excess lines:\n\n %s" % debug_diff) def testInterface(self): """Check whether the object follows the interface.""" self.failUnless( verifyObject(ITranslationFormatExporter, self.translation_exporter), "GettextPOExporter doesn't follow the interface", ) def testSupportedFormats(self): """Check that the exporter reports the correct formats.""" self.failUnlessEqual( self.translation_exporter.format, TranslationFileFormat.PO, "Expected GettextPOExporter to provide PO format " "but got %r instead." % self.translation_exporter.format, ) self.failUnlessEqual( self.translation_exporter.supported_source_formats, [TranslationFileFormat.PO, TranslationFileFormat.KDEPO], "Expected GettextPOExporter to support PO and KDEPO source " "formats but got %r instead." % (self.translation_exporter.supported_source_formats), ) def testGeneralExport(self): """Check different kind of messages export.""" pofile_cy = dedent( """ msgid "" msgstr "" "Project-Id-Version: foo\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: Kubla Kahn <*****@*****.**>\\n" "Language-Team: LANGUAGE <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=UTF-8\\n" "Content-Transfer-Encoding: 8bit\\n" "Plural-Forms: nplurals=4; plural=n==1 ? 0 : n==2 ? 1 : (n != 8 || n != 11) ? " "2 : 3;\\n" msgid "foo" msgid_plural "foos" msgstr[0] "cy-F001" msgstr[1] "cy-F002" msgstr[2] "" msgstr[3] "" #, fuzzy #| msgid "zog" msgid "zig" msgstr "zag" #, c-format msgid "zip" msgstr "zap" # tove #. borogove #: rath msgid "zog" msgstr "zug" #~ msgid "zot" #~ msgstr "zat" """ ) cy_translation_file = self.parser.parse(pofile_cy) cy_translation_file.is_template = False cy_translation_file.language_code = "cy" cy_translation_file.path = "po/cy.po" cy_translation_file.translation_domain = "testing" storage = ExportFileStorage() self.translation_exporter.exportTranslationFile(cy_translation_file, storage) self._compareImportAndExport(pofile_cy.strip(), storage.export().read().strip()) def testObsoleteExport(self): """Check how obsoleted messages are exported.""" pofile_eo = dedent( """ msgid "" msgstr "" "Project-Id-Version: Kumquats 1.0\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: L.L. Zamenhoff <*****@*****.**>\\n" "Language-Team: Esperanto <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=UTF-8\\n" "Content-Transfer-Encoding: 8bit\\n" # Foo bar. #, c-format #: src/foo.c #| msgid "zog" msgid "zig" msgstr "zag" """ ) pofile_eo_obsolete = dedent( """ msgid "" msgstr "" "Project-Id-Version: Kumquats 1.0\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: L.L. Zamenhoff <*****@*****.**>\\n" "Language-Team: Esperanto <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=UTF-8\\n" "Content-Transfer-Encoding: 8bit\\n" # Foo bar. #, c-format #~| msgid "zog" #~ msgid "zig" #~ msgstr "zag" """ ) eo_translation_file = self.parser.parse(pofile_eo) eo_translation_file.is_template = False eo_translation_file.language_code = "eo" eo_translation_file.path = "po/eo.po" eo_translation_file.translation_domain = "testing" eo_translation_file.messages[0].is_obsolete = True storage = ExportFileStorage() self.translation_exporter.exportTranslationFile(eo_translation_file, storage) self._compareImportAndExport(pofile_eo_obsolete.strip(), storage.export().read().strip()) def testEncodingExport(self): """Test that PO headers specifying character sets are respected.""" def compare(self, pofile): "Compare the original text with the exported one." "" # This is the word 'Japanese' in Japanese, in Unicode. nihongo_unicode = u"\u65e5\u672c\u8a9e" translation_file = self.parser.parse(pofile) translation_file.is_template = False translation_file.language_code = "ja" translation_file.path = "po/ja.po" translation_file.translation_domain = "testing" # We are sure that 'Japanese' is correctly stored as Unicode so # we are sure the exporter does its job instead of just export # what was imported. self.assertEqual(translation_file.messages[0].translations, [nihongo_unicode]) storage = ExportFileStorage() self.translation_exporter.exportTranslationFile(translation_file, storage) self._compareImportAndExport(pofile.strip(), storage.export().read().strip()) # File representing the same PO file three times. Each is identical # except for the charset declaration in the header. pofiles = [ dedent( """ msgid "" msgstr "" "Project-Id-Version: foo\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: Kubla Kahn <*****@*****.**>\\n" "Language-Team: LANGUAGE <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=UTF-8\\n" "Content-Transfer-Encoding: 8bit\\n" msgid "Japanese" msgstr "\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e" """ ), dedent( """ msgid "" msgstr "" "Project-Id-Version: foo\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: Kubla Kahn <*****@*****.**>\\n" "Language-Team: LANGUAGE <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=Shift-JIS\\n" "Content-Transfer-Encoding: 8bit\\n" msgid "Japanese" msgstr "\x93\xfa\x96\x7b\x8c\xea" """ ), dedent( """ msgid "" msgstr "" "Project-Id-Version: foo\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: Kubla Kahn <*****@*****.**>\\n" "Language-Team: LANGUAGE <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=EUC-JP\\n" "Content-Transfer-Encoding: 8bit\\n" msgid "Japanese" msgstr "\xc6\xfc\xcb\xdc\xb8\xec" """ ), ] for pofile in pofiles: compare(self, pofile) def _testBrokenEncoding(self, pofile_content): translation_file = self.parser.parse(pofile_content % {"charset": "ISO-8859-15", "special": "\xe1"}) translation_file.is_template = False translation_file.language_code = "es" translation_file.path = "po/es.po" translation_file.translation_domain = "testing" # Force the export as ASCII, it will not be possible because # translation is not available in that encoding and thus, we should # get an export in UTF-8. translation_file.header.charset = "ASCII" storage = ExportFileStorage() self.translation_exporter.exportTranslationFile(translation_file, storage) self._compareImportAndExport( pofile_content.strip() % {"charset": "UTF-8", "special": "\xc3\xa1"}, storage.export().read().strip() ) def testBrokenEncodingExport(self): """Test what happens when the content and the encoding don't agree. If a pofile fails to encode using the character set specified in the header, the header should be changed to specify to UTF-8 and the pofile exported accordingly. """ pofile_content = dedent( """ msgid "" msgstr "" "Project-Id-Version: foo\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: Kubla Kahn <*****@*****.**>\\n" "Language-Team: LANGUAGE <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=%(charset)s\\n" "Content-Transfer-Encoding: 8bit\\n" msgid "a" msgstr "%(special)s" """ ) self._testBrokenEncoding(pofile_content) def testBrokenEncodingHeader(self): """A header field might require a different encoding, too. This usually happens if the Last-Translator name contains non-ascii characters. If a pofile fails to encode using the character set specified in the header, the header should be changed to specify to UTF-8 and the pofile exported accordingly. """ pofile_content = dedent( """ msgid "" msgstr "" "Project-Id-Version: foo\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: Kubla K%(special)shn <*****@*****.**>\\n" "Language-Team: LANGUAGE <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=%(charset)s\\n" "Content-Transfer-Encoding: 8bit\\n" msgid "a" msgstr "b" """ ) self._testBrokenEncoding(pofile_content) def testIncompletePluralMessage(self): """Test export correctness for partial plural messages.""" pofile = dedent( """ msgid "" msgstr "" "Project-Id-Version: foo\\n" "Report-Msgid-Bugs-To: \\n" "POT-Creation-Date: 2007-07-09 03:39+0100\\n" "PO-Revision-Date: 2001-09-09 01:46+0000\\n" "Last-Translator: Kubla Kahn <*****@*****.**>\\n" "Language-Team: LANGUAGE <*****@*****.**>\\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=UTF-8\\n" "Content-Transfer-Encoding: 8bit\\n" "Plural-Forms: nplurals=2; plural=(n != 1);\\n" msgid "1 dead horse" msgid_plural "%%d dead horses" msgstr[0] "ning\xc3\xban caballo muerto" %s""" ) translation_file = self.parser.parse(pofile % ("")) translation_file.is_template = False translation_file.language_code = "es" translation_file.path = "po/es.po" translation_file.translation_domain = "testing" storage = ExportFileStorage() self.translation_exporter.exportTranslationFile(translation_file, storage) self._compareImportAndExport(pofile.strip() % 'msgstr[1] ""', storage.export().read().strip()) def testClashingSingularMsgIds(self): # We don't accept it in gettext imports directly, since it's not # valid gettext, but it's possible for our database to hold # messages that differ only in msgid_plural. In gettext those # would be considered equal, so we can't export them. Only the # first of the two messages is exported. template = self.factory.makePOTemplate() self.factory.makePOTMsgSet(template, singular="%d foo", plural="%d foos", sequence=1) self.factory.makePOTMsgSet(template, singular="%d foo", plural="%d foox", sequence=2) exported_file = template.export() # The "foos" (as opposed to "foox") tells us that the exporter # has picked the first message for export. expected_output = dedent( """ msgid "%d foo" msgid_plural "%d foos" msgstr[0] "" msgstr[1] "" """ ).strip() body = exported_file.split("\n\n", 1)[1].strip() self.assertEqual(body, expected_output) def testObsoleteMessageYieldsToNonObsoleteClashingOne(self): # When an obsolete message and a non-obsolete message in the # same POFile have identical identifying information except # msgid_plural (which Launchpad considers part of the message's # identifying information but gettext does not), only the # non-obsolete one is exported. template = self.factory.makePOTemplate() obsolete_message = self.factory.makePOTMsgSet(template, singular="%d goo", plural="%d goos", sequence=0) current_message = self.factory.makePOTMsgSet(template, singular="%d goo", plural="%d gooim", sequence=1) pofile = self.factory.makePOFile(potemplate=template, language_code="nl") self.factory.makeCurrentTranslationMessage( pofile=pofile, potmsgset=obsolete_message, translations=["%d splut", "%d splutjes"] ) self.factory.makeCurrentTranslationMessage( pofile=pofile, potmsgset=current_message, translations=["%d gargl", "%d garglii"] ) exported_file = pofile.export() # The "gooim" (as opposed to "goos") tells us that the exporter # has picked the non-obsolete message for export. The "gargl" # and "garglii" tell us we're not just getting the msgid from # the non-obsolete message, but the translations as well. expected_output = dedent( """ msgid "%d goo" msgid_plural "%d gooim" msgstr[0] "%d gargl" msgstr[1] "%d garglii" """ ).strip() body = exported_file.split("\n\n", 1)[1].strip() self.assertEqual(expected_output, body) def test_strip_last_newline(self): # `strip_last_newline` strips only the last newline. self.assertEqual("text\n", strip_last_newline("text\n\n")) self.assertEqual("text\nx", strip_last_newline("text\nx")) self.assertEqual("text", strip_last_newline("text")) # It supports '\r' as well (Mac-style). self.assertEqual("text", strip_last_newline("text\r")) # And DOS-style '\r\n'. self.assertEqual("text", strip_last_newline("text\r\n")) # With weird combinations, it strips only the last # newline-indicating character. self.assertEqual("text\n", strip_last_newline("text\n\r")) def test_comments_text_representation_multiline(self): # Comments with newlines should be correctly exported. data = TranslationMessageData() data.comment = "Line One\nLine Two" self.assertEqual("#Line One\n#Line Two", comments_text_representation(data)) # It works the same when there's a final newline as well. data.comment = "Line One\nLine Two\n" self.assertEqual("#Line One\n#Line Two", comments_text_representation(data)) # And similar processing happens for source comments. data = TranslationMessageData() data.source_comment = "Line One\nLine Two" self.assertEqual("#. Line One\n#. Line Two", comments_text_representation(data)) # It works the same when there's a final newline as well. data.source_comment = "Line One\nLine Two\n" self.assertEqual("#. Line One\n#. Line Two", comments_text_representation(data)) def test_export_handles_empty_files(self): # Exporting an empty gettext file does not break the exporter. # The output does contain one message: the header. output = self.factory.makePOFile("nl").export() self.assertTrue(output.startswith("# Dutch translation for ")) self.assertEqual(1, output.count("msgid"))