def test_string_from_plaintext(self): string = StringValue.from_plaintext( "This is a test, \"foo\" bar 'baz'.!?;:", ) self.assertEqual( string.data, "This is a test, \"foo\" bar 'baz'.!?;:", )
def test_br_tag_is_treated_as_inline_tag(self): template, strings = extract_strings( "<p><b>Foo <i>Bar<br/>Baz</i></b></p>") self.assertHTMLEqual(template, '<p><b><text position="0"></text></b></p>') self.assertEqual(strings, [StringValue.from_html("Foo <i>Bar<br/>Baz</i>")])
def from_source_html(cls, path, html, **kwargs): """ Initialises a StringSegmentValue from a HTML string. Args: path (str): The content path of the segment. html (str): The HTML value of the segment. order (int, optional): The index that this segment appears on a page. """ string, attrs = StringValue.from_source_html(html) return cls(path, string, attrs=attrs, **kwargs)
def test_block_tags_not_allowed(self): with self.assertRaises(ValueError) as e: StringValue.from_translated_html("<p>Foo bar baz</p>") self.assertIsInstance(e.exception, ValueError) self.assertEqual( e.exception.args, ("<p> tag is not allowed. Strings can only contain standard HTML inline tags (such as <b>, <a>)", ), ) with self.assertRaises(ValueError) as e: StringValue.from_translated_html("<img/>") self.assertIsInstance(e.exception, ValueError) self.assertEqual( e.exception.args, ("<img> tag is not allowed. Strings can only contain standard HTML inline tags (such as <b>, <a>)", ), )
def test_import_po(self): obsolete_string = String.from_value( self.en_locale, StringValue("This is an obsolete string")) StringTranslation.objects.create( translation_of=obsolete_string, context=TranslationContext.objects.get(path="test_charfield"), locale=self.fr_locale, data="Ceci est une chaîne obsolète", ) po = polib.POFile(wrapwidth=200) po.metadata = { "POT-Creation-Date": str(timezone.now()), "MIME-Version": "1.0", "Content-Type": "text/plain; charset=utf-8", "X-WagtailLocalize-TranslationID": str(self.translation.uuid), } po.append( polib.POEntry( msgid="This is some test content", msgctxt="test_charfield", msgstr="Contenu de test", )) po.append( polib.POEntry( msgid="This is an obsolete string", msgctxt="test_charfield", msgstr="C'est encore une chaîne obsolète", obsolete=True, )) warnings = self.translation.import_po(po) self.assertEqual(warnings, []) translation = StringTranslation.objects.get( translation_of__data="This is some test content") self.assertEqual(translation.context, TranslationContext.objects.get(path="test_charfield")) self.assertEqual(translation.locale, self.fr_locale) self.assertEqual(translation.data, "Contenu de test") # Obsolete strings still get updated translation = StringTranslation.objects.get( translation_of__data="This is an obsolete string") self.assertEqual(translation.context, TranslationContext.objects.get(path="test_charfield")) self.assertEqual(translation.locale, self.fr_locale) self.assertEqual( translation.data, "C'est encore une chaîne obsolète", )
def test_warnings(self): String.from_value( self.en_locale, StringValue( "This string exists in the database but isn't relevant to this object" ), ) po = polib.POFile(wrapwidth=200) po.metadata = { "POT-Creation-Date": str(timezone.now()), "MIME-Version": "1.0", "Content-Type": "text/plain; charset=utf-8", "X-WagtailLocalize-TranslationID": str(self.translation.uuid), } po.append( polib.POEntry( msgid= "This string exists in the database but isn't relevant to this object", msgctxt="test_charfield", msgstr="Contenu de test", )) po.append( polib.POEntry( msgid="This string doesn't exist", msgctxt="test_charfield", msgstr="Contenu de test", )) po.append( polib.POEntry( msgid="This is some test content", msgctxt="invalidcontext", msgstr="Contenu de test", )) warnings = self.translation.import_po(po) self.assertEqual( warnings, [ StringNotUsedInContext( 0, "This string exists in the database but isn't relevant to this object", "test_charfield", ), UnknownString(1, "This string doesn't exist"), UnknownContext(2, "invalidcontext"), ], )
def translate(self, source_locale, target_locale, strings): response = requests.post('https://api.deepl.com/v2/translate', { 'auth_key': self.options['AUTH_KEY'], 'text': [string.data for string in strings], 'tag_handling': 'xml', 'source_lang': language_code(source_locale.language_code), 'target_lang': language_code(target_locale.language_code, is_target=True), }) return { string: StringValue(translation['text']) for string, translation in zip(strings, response.json()['translations']) }
def test_extract_strings_2(self): template, strings = extract_strings(""" <h1>Foo bar baz</h1> <p>This is a paragraph. <b>This is some bold <i>and now italic</i></b> text</p> <p><script> this should be interpreted as text.</p> <ul> <li>List item one</li> <li><b>List item two</li> </ul> <img src="foo" alt="This bit isn't translatable"> """) self.assertHTMLEqual( template, """ <h1><text position="0"></text></h1> <p><text position="1"></text></p> <p><text position="2"></text></p> <ul> <li><text position="3"></text></li> <li><b><text position="4"></text></b></li> </ul> <img alt="This bit isn\'t translatable" src="foo"> """, ) self.assertEqual( strings, [ StringValue.from_html("Foo bar baz"), StringValue.from_html( "This is a paragraph. <b>This is some bold <i>and now italic</i></b> text" ), StringValue.from_html( "<script> this should be interpreted as text."), StringValue.from_html("List item one"), StringValue.from_html("List item two"), ], )
def test_translate_html(self): string, attrs = StringValue.from_source_html( '<a href="https://en.wikipedia.org/wiki/World">Hello world!</a>. <b>This is a test</b>.' ) translations = DummyTranslator({}).translate(self.english_locale, self.french_locale, [string]) self.assertEqual( translations[string].render_html(attrs), '.<b>test a is This</b>. <a href="https://en.wikipedia.org/wiki/World">world! Hello</a>', )
def test_restore_strings(self): html = restore_strings( """ <p><text position="0"></text></p> <p><text position="1"></text></p> """, [ StringValue.from_html( '<b>Bread</b>\xa0is a\xa0<a href="https://en.wikipedia.org/wiki/Staple_food">staple food</a>\xa0prepared from a\xa0<a href="https://en.wikipedia.org/wiki/Dough">dough</a>\xa0of\xa0<a href="https://en.wikipedia.org/wiki/Flour">flour</a>\xa0and\xa0<a href="https://en.wikipedia.org/wiki/Water">water</a>, usually by\xa0<a href="https://en.wikipedia.org/wiki/Baking">baking</a>. Throughout recorded history it has been popular around the world and is one of the oldest artificial foods, having been of importance since the dawn of\xa0<a href="https://en.wikipedia.org/wiki/Agriculture#History">agriculture</a>.' ), StringValue.from_html( 'Proportions of types of flour and other ingredients vary widely, as do modes of preparation. As a result, types, shapes, sizes, and textures of breads differ around the world. Bread may be\xa0<a href="https://en.wikipedia.org/wiki/Leaven">leavened</a>\xa0by processes such as reliance on naturally occurring\xa0<a href="https://en.wikipedia.org/wiki/Sourdough">sourdough</a>\xa0microbes, chemicals, industrially produced yeast, or high-pressure aeration. Some bread is cooked before it can leaven, including for traditional or religious reasons. Non-cereal ingredients such as fruits, nuts and fats may be included. Commercial bread commonly contains additives to improve flavor, texture, color, shelf life, and ease of manufacturing.' ), ], ) self.assertHTMLEqual( """ <p><b>Bread</b>\xa0is a\xa0<a href="https://en.wikipedia.org/wiki/Staple_food">staple food</a>\xa0prepared from a\xa0<a href="https://en.wikipedia.org/wiki/Dough">dough</a>\xa0of\xa0<a href="https://en.wikipedia.org/wiki/Flour">flour</a>\xa0and\xa0<a href="https://en.wikipedia.org/wiki/Water">water</a>, usually by\xa0<a href="https://en.wikipedia.org/wiki/Baking">baking</a>. Throughout recorded history it has been popular around the world and is one of the oldest artificial foods, having been of importance since the dawn of\xa0<a href="https://en.wikipedia.org/wiki/Agriculture#History">agriculture</a>.</p> <p>Proportions of types of flour and other ingredients vary widely, as do modes of preparation. As a result, types, shapes, sizes, and textures of breads differ around the world. Bread may be\xa0<a href="https://en.wikipedia.org/wiki/Leaven">leavened</a>\xa0by processes such as reliance on naturally occurring\xa0<a href="https://en.wikipedia.org/wiki/Sourdough">sourdough</a>\xa0microbes, chemicals, industrially produced yeast, or high-pressure aeration. Some bread is cooked before it can leaven, including for traditional or religious reasons. Non-cereal ingredients such as fruits, nuts and fats may be included. Commercial bread commonly contains additives to improve flavor, texture, color, shelf life, and ease of manufacturing.</p> """, html, )
def test_string_from_plaintext(self): string = StringValue.from_plaintext("This is a test <Foo> bar 'baz'", ) # Django 2.x HTML escape function escapes quotes differently if DJANGO_VERSION >= (3, 0): self.assertEqual( string.data, "This is a test <Foo> bar 'baz'", ) else: self.assertEqual( string.data, "This is a test <Foo> bar 'baz'", )
def test_translate_text(self): translations = DummyTranslator({}).translate(self.english_locale, self.french_locale, [ StringValue("Hello world!"), StringValue("This is a sentence. This is another sentence."), ]) self.assertEqual(translations, { StringValue("Hello world!"): StringValue("world! Hello"), StringValue("This is a sentence. This is another sentence."): StringValue("sentence. another is This sentence. a is This"), })
def translate(self, source_locale, target_locale, strings): translator = googletrans.Translator() google_translations = translator.translate( [string.render_text() for string in strings], src=language_code(source_locale.language_code), dest=language_code(target_locale.language_code), ) translations = { translation.origin: translation.text for translation in google_translations } return { string: StringValue.from_plaintext(translations[string.render_text()]) for string in strings }
def translate(self, source_locale, target_locale, strings): client = translate.TranslationServiceClient() project_id = self.options['PROJECT_ID'] location = self.options.get('LOCATION', 'global') response = client.translate_text( request={ 'parent': f'projects/{project_id}/locations/{location}', 'contents': [string.data for string in strings], 'mime_type': 'text/html', 'source_language_code': source_locale.language_code, 'target_language_code': target_locale.language_code, }) return { string: StringValue(translation.translated_text) for string, translation in zip(strings, response.translations) }
def translate(self, source_locale, target_locale, strings): response = requests.post( "https://api.deepl.com/v2/translate", { "auth_key": self.options["AUTH_KEY"], "text": [string.data for string in strings], "tag_handling": "xml", "source_lang": language_code(source_locale.language_code), "target_lang": language_code(target_locale.language_code, is_target=True), }, ) return { string: StringValue(translation["text"]) for string, translation in zip(strings, response.json()["translations"]) }
TemplateSegmentValue( "", "html", '<h1><text position="0"></text></h1><p><text position="1"></text></p><ul><li><text position="2"></text></li></ul>', 3, order=9, ), StringSegmentValue("", "Ceci est une rubrique", order=10), StringSegmentValue.from_source_html( "", 'Ceci est un paragraphe. <foo> <b>Texte en gras</b>', order=11, ), StringSegmentValue( "", StringValue('<a id="a1">Ceci est un lien</a>'), attrs={ "a1": {"href": "http://example.com"} }, order=12, ), ] RICH_TEXT_TEST_OUTPUT = '<h1>Ceci est une rubrique</h1><p>Ceci est un paragraphe. <foo> <b>Texte en gras</b></p><ul><li><a href="http://example.com">Ceci est un lien</a></li></ul>' class TestSegmentIngestion(TestCase): def setUp(self): self.src_locale = Locale.get_default() self.locale = Locale.objects.create(language_code="fr")
def test_unicode_chars_not_escaped(self): # unicode characters should not be escaped as this would be horrible for translators! string = StringValue.from_plaintext("セキレイ") self.assertEqual(string.data, "セキレイ")
def translate(self, source_locale, target_locale, strings): return {string: StringValue(translate_html(string.data)) for string in strings}
def test_br_tags_converted_to_newlines(self): string = StringValue("foo<br>bar<br>baz") self.assertEqual(string.render_text(), "foo\nbar\nbaz") string = StringValue("<br/><b>foo</b><br/><i>bar</i><br/>") self.assertEqual(string.render_text(), "\nfoo\nbar\n")
def test_special_chars_unescaped(self): string = StringValue("<b>foo</b><i> & bar</i>") self.assertEqual(string.render_text(), "foo & bar")
def test_special_chars_escaped(self): string = StringValue.from_plaintext("foo & bar") self.assertEqual(string.data, "foo & bar")
def from_source_html(cls, path, html, **kwargs): string, attrs = StringValue.from_source_html(html) return cls(path, string, attrs=attrs, **kwargs)
def test_newlines_converted_to_br_tags(self): string = StringValue.from_plaintext("foo\nbar\nbaz") self.assertEqual(string.data, "foo<br>bar<br>baz") string = StringValue.from_plaintext("\nfoo\nbar\n") self.assertEqual(string.data, "<br>foo<br>bar<br>")
RICH_TEXT_TEST_OUTPUT = [ TemplateSegmentValue( "", "html", '<h1><text position="0"></text></h1><p><text position="1"></text></p><ul><li><text position="2"></text></li></ul>', 3, ), StringSegmentValue("", "This is a heading"), StringSegmentValue.from_source_html( "", "This is a paragraph. <foo> <b>Bold text</b>", ), StringSegmentValue( "", StringValue('<a id="a1">This is a link</a>.'), attrs={"a1": { "href": "http://example.com" }}, ), OverridableSegmentValue("'http://example.com'", "http://example.com"), ] class TestSegmentExtraction(TestCase): def test_charfield(self): page = make_test_page(test_charfield="Test content") segments = extract_segments(page) self.assertEqual( segments, [StringSegmentValue("test_charfield", "Test content")])