def test_string_from_plaintext(self):
string = StringValue.from_plaintext(
"This is a test, \"foo\" bar 'baz'.!?;:", )
self.assertEqual(
string.data,
"This is a test, \"foo\" bar 'baz'.!?;:",
)
def test_br_tag_is_treated_as_inline_tag(self):
template, strings = extract_strings(
"<p><b>Foo <i>Bar<br/>Baz</i></b></p>")
self.assertHTMLEqual(template,
'<p><b><text position="0"></text></b></p>')
self.assertEqual(strings,
[StringValue.from_html("Foo <i>Bar<br/>Baz</i>")])
def from_source_html(cls, path, html, **kwargs):
"""
Initialises a StringSegmentValue from a HTML string.
Args:
path (str): The content path of the segment.
html (str): The HTML value of the segment.
order (int, optional): The index that this segment appears on a page.
"""
string, attrs = StringValue.from_source_html(html)
return cls(path, string, attrs=attrs, **kwargs)
def test_block_tags_not_allowed(self):
with self.assertRaises(ValueError) as e:
StringValue.from_translated_html("<p>Foo bar baz</p>")
self.assertIsInstance(e.exception, ValueError)
self.assertEqual(
e.exception.args,
("<p> tag is not allowed. Strings can only contain standard HTML inline tags (such as <b>, <a>)",
),
)
with self.assertRaises(ValueError) as e:
StringValue.from_translated_html("<img/>")
self.assertIsInstance(e.exception, ValueError)
self.assertEqual(
e.exception.args,
("<img> tag is not allowed. Strings can only contain standard HTML inline tags (such as <b>, <a>)",
),
)
def test_import_po(self):
obsolete_string = String.from_value(
self.en_locale, StringValue("This is an obsolete string"))
StringTranslation.objects.create(
translation_of=obsolete_string,
context=TranslationContext.objects.get(path="test_charfield"),
locale=self.fr_locale,
data="Ceci est une chaîne obsolète",
)
po = polib.POFile(wrapwidth=200)
po.metadata = {
"POT-Creation-Date": str(timezone.now()),
"MIME-Version": "1.0",
"Content-Type": "text/plain; charset=utf-8",
"X-WagtailLocalize-TranslationID": str(self.translation.uuid),
}
po.append(
polib.POEntry(
msgid="This is some test content",
msgctxt="test_charfield",
msgstr="Contenu de test",
))
po.append(
polib.POEntry(
msgid="This is an obsolete string",
msgctxt="test_charfield",
msgstr="C'est encore une chaîne obsolète",
obsolete=True,
))
warnings = self.translation.import_po(po)
self.assertEqual(warnings, [])
translation = StringTranslation.objects.get(
translation_of__data="This is some test content")
self.assertEqual(translation.context,
TranslationContext.objects.get(path="test_charfield"))
self.assertEqual(translation.locale, self.fr_locale)
self.assertEqual(translation.data, "Contenu de test")
# Obsolete strings still get updated
translation = StringTranslation.objects.get(
translation_of__data="This is an obsolete string")
self.assertEqual(translation.context,
TranslationContext.objects.get(path="test_charfield"))
self.assertEqual(translation.locale, self.fr_locale)
self.assertEqual(
translation.data,
"C'est encore une chaîne obsolète",
)
def test_warnings(self):
String.from_value(
self.en_locale,
StringValue(
"This string exists in the database but isn't relevant to this object"
),
)
po = polib.POFile(wrapwidth=200)
po.metadata = {
"POT-Creation-Date": str(timezone.now()),
"MIME-Version": "1.0",
"Content-Type": "text/plain; charset=utf-8",
"X-WagtailLocalize-TranslationID": str(self.translation.uuid),
}
po.append(
polib.POEntry(
msgid=
"This string exists in the database but isn't relevant to this object",
msgctxt="test_charfield",
msgstr="Contenu de test",
))
po.append(
polib.POEntry(
msgid="This string doesn't exist",
msgctxt="test_charfield",
msgstr="Contenu de test",
))
po.append(
polib.POEntry(
msgid="This is some test content",
msgctxt="invalidcontext",
msgstr="Contenu de test",
))
warnings = self.translation.import_po(po)
self.assertEqual(
warnings,
[
StringNotUsedInContext(
0,
"This string exists in the database but isn't relevant to this object",
"test_charfield",
),
UnknownString(1, "This string doesn't exist"),
UnknownContext(2, "invalidcontext"),
],
)
def translate(self, source_locale, target_locale, strings):
response = requests.post('https://api.deepl.com/v2/translate', {
'auth_key': self.options['AUTH_KEY'],
'text': [string.data for string in strings],
'tag_handling': 'xml',
'source_lang': language_code(source_locale.language_code),
'target_lang': language_code(target_locale.language_code, is_target=True),
})
return {
string: StringValue(translation['text'])
for string, translation in zip(strings, response.json()['translations'])
}
def test_extract_strings_2(self):
template, strings = extract_strings("""
<h1>Foo bar baz</h1>
<p>This is a paragraph. <b>This is some bold <i>and now italic</i></b> text</p>
<p><script> this should be interpreted as text.</p>
<ul>
<li>List item one</li>
<li><b>List item two</li>
</ul>
<img src="foo" alt="This bit isn't translatable">
""")
self.assertHTMLEqual(
template,
"""
<h1><text position="0"></text></h1>
<p><text position="1"></text></p>
<p><text position="2"></text></p>
<ul>
<li><text position="3"></text></li>
<li><b><text position="4"></text></b></li>
</ul>
<img alt="This bit isn\'t translatable" src="foo">
""",
)
self.assertEqual(
strings,
[
StringValue.from_html("Foo bar baz"),
StringValue.from_html(
"This is a paragraph. <b>This is some bold <i>and now italic</i></b> text"
),
StringValue.from_html(
"<script> this should be interpreted as text."),
StringValue.from_html("List item one"),
StringValue.from_html("List item two"),
],
)
def test_translate_html(self):
string, attrs = StringValue.from_source_html(
'<a href="https://en.wikipedia.org/wiki/World">Hello world!</a>. <b>This is a test</b>.'
)
translations = DummyTranslator({}).translate(self.english_locale,
self.french_locale,
[string])
self.assertEqual(
translations[string].render_html(attrs),
'.<b>test a is This</b>. <a href="https://en.wikipedia.org/wiki/World">world! Hello</a>',
)
def test_restore_strings(self):
html = restore_strings(
"""
<p><text position="0"></text></p>
<p><text position="1"></text></p>
""",
[
StringValue.from_html(
'<b>Bread</b>\xa0is a\xa0<a href="https://en.wikipedia.org/wiki/Staple_food">staple food</a>\xa0prepared from a\xa0<a href="https://en.wikipedia.org/wiki/Dough">dough</a>\xa0of\xa0<a href="https://en.wikipedia.org/wiki/Flour">flour</a>\xa0and\xa0<a href="https://en.wikipedia.org/wiki/Water">water</a>, usually by\xa0<a href="https://en.wikipedia.org/wiki/Baking">baking</a>. Throughout recorded history it has been popular around the world and is one of the oldest artificial foods, having been of importance since the dawn of\xa0<a href="https://en.wikipedia.org/wiki/Agriculture#History">agriculture</a>.'
),
StringValue.from_html(
'Proportions of types of flour and other ingredients vary widely, as do modes of preparation. As a result, types, shapes, sizes, and textures of breads differ around the world. Bread may be\xa0<a href="https://en.wikipedia.org/wiki/Leaven">leavened</a>\xa0by processes such as reliance on naturally occurring\xa0<a href="https://en.wikipedia.org/wiki/Sourdough">sourdough</a>\xa0microbes, chemicals, industrially produced yeast, or high-pressure aeration. Some bread is cooked before it can leaven, including for traditional or religious reasons. Non-cereal ingredients such as fruits, nuts and fats may be included. Commercial bread commonly contains additives to improve flavor, texture, color, shelf life, and ease of manufacturing.'
),
],
)
self.assertHTMLEqual(
"""
<p><b>Bread</b>\xa0is a\xa0<a href="https://en.wikipedia.org/wiki/Staple_food">staple food</a>\xa0prepared from a\xa0<a href="https://en.wikipedia.org/wiki/Dough">dough</a>\xa0of\xa0<a href="https://en.wikipedia.org/wiki/Flour">flour</a>\xa0and\xa0<a href="https://en.wikipedia.org/wiki/Water">water</a>, usually by\xa0<a href="https://en.wikipedia.org/wiki/Baking">baking</a>. Throughout recorded history it has been popular around the world and is one of the oldest artificial foods, having been of importance since the dawn of\xa0<a href="https://en.wikipedia.org/wiki/Agriculture#History">agriculture</a>.</p>
<p>Proportions of types of flour and other ingredients vary widely, as do modes of preparation. As a result, types, shapes, sizes, and textures of breads differ around the world. Bread may be\xa0<a href="https://en.wikipedia.org/wiki/Leaven">leavened</a>\xa0by processes such as reliance on naturally occurring\xa0<a href="https://en.wikipedia.org/wiki/Sourdough">sourdough</a>\xa0microbes, chemicals, industrially produced yeast, or high-pressure aeration. Some bread is cooked before it can leaven, including for traditional or religious reasons. Non-cereal ingredients such as fruits, nuts and fats may be included. Commercial bread commonly contains additives to improve flavor, texture, color, shelf life, and ease of manufacturing.</p>
""",
html,
)
def test_string_from_plaintext(self):
string = StringValue.from_plaintext("This is a test <Foo> bar 'baz'", )
# Django 2.x HTML escape function escapes quotes differently
if DJANGO_VERSION >= (3, 0):
self.assertEqual(
string.data,
"This is a test <Foo> bar 'baz'",
)
else:
self.assertEqual(
string.data,
"This is a test <Foo> bar 'baz'",
)
def test_translate_text(self):
translations = DummyTranslator({}).translate(self.english_locale, self.french_locale, [
StringValue("Hello world!"),
StringValue("This is a sentence. This is another sentence."),
])
self.assertEqual(translations, {
StringValue("Hello world!"): StringValue("world! Hello"),
StringValue("This is a sentence. This is another sentence."): StringValue("sentence. another is This sentence. a is This"),
})
def translate(self, source_locale, target_locale, strings):
translator = googletrans.Translator()
google_translations = translator.translate(
[string.render_text() for string in strings],
src=language_code(source_locale.language_code),
dest=language_code(target_locale.language_code),
)
translations = {
translation.origin: translation.text
for translation in google_translations
}
return {
string:
StringValue.from_plaintext(translations[string.render_text()])
for string in strings
}
def translate(self, source_locale, target_locale, strings):
client = translate.TranslationServiceClient()
project_id = self.options['PROJECT_ID']
location = self.options.get('LOCATION', 'global')
response = client.translate_text(
request={
'parent': f'projects/{project_id}/locations/{location}',
'contents': [string.data for string in strings],
'mime_type': 'text/html',
'source_language_code': source_locale.language_code,
'target_language_code': target_locale.language_code,
})
return {
string: StringValue(translation.translated_text)
for string, translation in zip(strings, response.translations)
}
def translate(self, source_locale, target_locale, strings):
response = requests.post(
"https://api.deepl.com/v2/translate",
{
"auth_key":
self.options["AUTH_KEY"],
"text": [string.data for string in strings],
"tag_handling":
"xml",
"source_lang":
language_code(source_locale.language_code),
"target_lang":
language_code(target_locale.language_code, is_target=True),
},
)
return {
string: StringValue(translation["text"])
for string, translation in zip(strings,
response.json()["translations"])
}
TemplateSegmentValue(
"",
"html",
'<h1><text position="0"></text></h1><p><text position="1"></text></p><ul><li><text position="2"></text></li></ul>',
3,
order=9,
),
StringSegmentValue("", "Ceci est une rubrique", order=10),
StringSegmentValue.from_source_html(
"",
'Ceci est un paragraphe. <foo> <b>Texte en gras</b>',
order=11,
),
StringSegmentValue(
"",
StringValue('<a id="a1">Ceci est un lien</a>'),
attrs={
"a1": {"href": "http://example.com"}
},
order=12,
),
]
RICH_TEXT_TEST_OUTPUT = '<h1>Ceci est une rubrique</h1><p>Ceci est un paragraphe. <foo> <b>Texte en gras</b></p><ul><li><a href="http://example.com">Ceci est un lien</a></li></ul>'
class TestSegmentIngestion(TestCase):
def setUp(self):
self.src_locale = Locale.get_default()
self.locale = Locale.objects.create(language_code="fr")
def test_unicode_chars_not_escaped(self):
# unicode characters should not be escaped as this would be horrible for translators!
string = StringValue.from_plaintext("セキレイ")
self.assertEqual(string.data, "セキレイ")
def translate(self, source_locale, target_locale, strings):
return {string: StringValue(translate_html(string.data)) for string in strings}
def test_br_tags_converted_to_newlines(self):
string = StringValue("foo<br>bar<br>baz")
self.assertEqual(string.render_text(), "foo\nbar\nbaz")
string = StringValue("<br/><b>foo</b><br/><i>bar</i><br/>")
self.assertEqual(string.render_text(), "\nfoo\nbar\n")
def test_special_chars_unescaped(self):
string = StringValue("<b>foo</b><i> & bar</i>")
self.assertEqual(string.render_text(), "foo & bar")
def test_special_chars_escaped(self):
string = StringValue.from_plaintext("foo & bar")
self.assertEqual(string.data, "foo & bar")
def from_source_html(cls, path, html, **kwargs):
string, attrs = StringValue.from_source_html(html)
return cls(path, string, attrs=attrs, **kwargs)
def test_newlines_converted_to_br_tags(self):
string = StringValue.from_plaintext("foo\nbar\nbaz")
self.assertEqual(string.data, "foo<br>bar<br>baz")
string = StringValue.from_plaintext("\nfoo\nbar\n")
self.assertEqual(string.data, "<br>foo<br>bar<br>")
RICH_TEXT_TEST_OUTPUT = [
TemplateSegmentValue(
"",
"html",
'<h1><text position="0"></text></h1><p><text position="1"></text></p><ul><li><text position="2"></text></li></ul>',
3,
),
StringSegmentValue("", "This is a heading"),
StringSegmentValue.from_source_html(
"",
"This is a paragraph. <foo> <b>Bold text</b>",
),
StringSegmentValue(
"",
StringValue('<a id="a1">This is a link</a>.'),
attrs={"a1": {
"href": "http://example.com"
}},
),
OverridableSegmentValue("'http://example.com'", "http://example.com"),
]
class TestSegmentExtraction(TestCase):
def test_charfield(self):
page = make_test_page(test_charfield="Test content")
segments = extract_segments(page)
self.assertEqual(
segments, [StringSegmentValue("test_charfield", "Test content")])
