def test_finding_all_translations_for_a_group(self):
TestModel.objects.create()
translations = MasterTranslation.find_by_group("Test")
self.assertEqual(0, translations.count())
TestModel.objects.create(trans_with_group=TranslatableContent(text="Hello World!"))
translations = MasterTranslation.find_by_group("Test")
self.assertEqual(1, translations.count())
def test_finding_all_translations_for_a_group(self):
TranslatedModel.objects.create()
translations = MasterTranslation.find_by_group("Test")
self.assertEqual(0, translations.count())
TranslatedModel.objects.create(trans_with_group=TranslatableContent(text="Hello World!"))
translations = MasterTranslation.find_by_group("Test")
self.assertEqual(1, translations.count())
def test_pk_is_set_correctly(self):
mt1 = MasterTranslation.objects.create(text="Hello World!", language_code="en")
self.assertEqual(mt1.pk, MasterTranslation.generate_key("Hello World!", "", "en"))
mt2 = MasterTranslation.objects.create(text="Hello World!", language_code="fr")
self.assertEqual(mt2.pk, MasterTranslation.generate_key("Hello World!", "", "fr"))
# Make sure that it differs
self.assertNotEqual(mt1.pk, mt2.pk)
def test_pk_is_set_correctly(self):
mt1 = MasterTranslation.objects.create(text="Hello World!",
language_code="en")
self.assertEqual(
mt1.pk, MasterTranslation.generate_key("Hello World!", "", "en"))
mt2 = MasterTranslation.objects.create(text="Hello World!",
language_code="fr")
self.assertEqual(
mt2.pk, MasterTranslation.generate_key("Hello World!", "", "fr"))
# Make sure that it differs
self.assertNotEqual(mt1.pk, mt2.pk)
def test_load_str_po(self):
"""Test that a str with multibyte uft-8 chars can parsed correctly"""
MasterTranslation(text=u"This — that", language_code="en").save()
MasterTranslation(text=u"something something something something translate", language_code="en").save()
pofile = """%s
msgid "This — that"
msgstr "Deise — dass"
""" % POFILE
errors = import_translations_from_po(pofile, "de", "en")
self.assertEqual(errors, [])
def test_blocktrans_tag_with_variable_escaping(self):
text = '\n<a href="http://google.com">%(name)s</a> without group\n'
content = """
{% load fluent %}
{% blocktrans %}
<a href="http://google.com">{{ name }}</a> without group
{% endblocktrans %}
"""
results = parse_file(content, ".html")
expected = [
(text, '', '', DEFAULT_TRANSLATION_GROUP),
]
self.assertEqual(results, expected)
data = {'name': "Ola & Ola"}
rendered = Template(content).render(Context(data))
self.assertTrue('<a href="http://google.com">Ola & Ola</a> without group' in rendered)
# create master translation for the string and test it renders correctly
key = MasterTranslation.generate_key(text, '', 'en-us')
MasterTranslation.objects.create(
pk=key, text=text, language_code='en-us'
)
rendered = Template(content).render(Context(data))
self.assertTrue('<a href="http://google.com">Ola & Ola</a> without group' in rendered)
def test_blocktrans_tag_with_variable_and_group_escaping(self):
text = '\n<a href="http://google.com">%(name)s</a> in group\n'
content = """
{% load fluent %}
{% blocktrans group "public" %}
<a href="http://google.com">{{ name }}</a> in group
{% endblocktrans %}
"""
results = parse_file(content, ".html")
expected = [
(text, '', '', 'public'),
]
self.assertEqual(results, expected)
data = {'name': "Ola & Ola"}
rendered = Template(content).render(Context(data))
self.assertTrue('<a href="http://google.com">Ola & Ola</a> in group' in rendered)
# create master translation for the string and test it renders correctly
key = MasterTranslation.generate_key(text, '', 'en-us')
mt = MasterTranslation.objects.create(
pk=key, text=text, language_code='en-us'
)
mt.used_by_groups_in_code_or_templates = {'public'}
mt.save()
rendered = Template(content).render(Context(data))
self.assertTrue('<a href="http://google.com">Ola & Ola</a> in group' in rendered)
def test_blocktrans_tag_with_group(self):
text = "\nTest trans block with group\n"
content = """
{% load fluent %}
{% blocktrans group "public" %}
Test trans block with group
{% endblocktrans %}
"""
results = parse_file(content, ".html")
expected = [
(text, '', '', 'public'),
]
self.assertEqual(results, expected)
rendered = Template(content).render(Context())
self.assertTrue(text in rendered)
# create master translation for the string and test it renders correctly
key = MasterTranslation.generate_key(text, '', 'en-us')
mt = MasterTranslation.objects.create(
pk=key, text=text, language_code='en-us'
)
mt.used_by_groups_in_code_or_templates = {'public'}
mt.save()
rendered = Template(content).render(Context())
self.assertTrue("Test trans block with group" in rendered)
def test_export(self):
MasterTranslation(
text=u"Wave", hint="Hand gesture", language_code="en").save()
MasterTranslation(
text=u"Wave", hint="Oceanic", language_code="en").save()
MasterTranslation(
text="One message removed",
hint="Message(s) removed plural",
plural_text="%d messages deleted",
language_code="en").save()
MasterTranslation(
text=u"Something to translate", language_code="en").save()
MasterTranslation(
text=u"Product®™ — Special chars", language_code="en").save()
po_file = polib.pofile(unicode(
export_translations_to_po('en').content.decode('utf-8')))
self.assertEqual(EXPECTED_EXPORT_PO_FILE, po_file.__unicode__())
def _scan_list(marshall, scan_id, filenames):
# FIXME: Need to clean up the translations which aren't in use anymore
for filename in filenames:
# Redeploying to a new version can cause this
if not os.path.exists(filename):
continue
with open(filename) as f:
content = f.read()
results = parse_file(content, os.path.splitext(filename)[-1])
for text, plural, hint, group in results:
if not text:
logger.warn("Empty translation discovered: '{}', '{}', '{}', '{}'".format(text, plural, hint, group))
continue
with transaction.atomic(xg=True):
key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE)
try:
mt = MasterTranslation.objects.get(pk=key)
except MasterTranslation.DoesNotExist:
mt = MasterTranslation(
pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE
)
# By the very act of getting here, this is true
mt.used_in_code_or_templates = True
# If we last updated during this scan, then append, otherwise replace
if mt.last_updated_by_scan_uuid == scan_id:
mt.used_by_groups_in_code_or_templates.add(group)
else:
mt.used_by_groups_in_code_or_templates = { group }
mt.last_updated_by_scan_uuid = scan_id
mt.save()
with transaction.atomic():
marshall.refresh_from_db()
marshall.files_left_to_process -= len(filenames)
marshall.save()
def test_export_handles_language_region_codes(self):
# Fluent should handle 'en-US' as a language code. Previously
# `export_translations_to_po('en-US')` would raise KeyError.
MasterTranslation(text=u'Foo', hint=u'Bar', language_code='en').save()
result = unicode(export_translations_to_po('en-US'))
expected = (
u'Content-Type: text/plain\r\n'
u'Content-Disposition: attachment; filename=django.po\r\n\r\n'
u'#. Bar\n'
u'msgctxt "Bar"\n'
u'msgid "Foo"\n'
u'msgstr "Foo"\n'
)
self.assertEqual(result, expected)
def test_trans_tag_without_group(self):
text = "Test trans string without group"
content = """
{% load fluent %}
{% trans "Test trans string without group" %}
"""
results = parse_file(content, ".html")
expected = [(text, '', '', DEFAULT_TRANSLATION_GROUP)]
self.assertEqual(results, expected)
rendered = Template(content).render(Context())
self.assertTrue(text in rendered)
# create master translation for the string and test it renders correctly
key = MasterTranslation.generate_key(text, '', 'en-us')
MasterTranslation.objects.create(
pk=key, text=text, language_code='en-us'
)
rendered = Template(content).render(Context())
self.assertTrue("Test trans string without group" in rendered)
def _scan_list(marshall, scan_id, filenames):
# FIXME: Need to clean up the translations which aren't in use anymore
for filename in filenames:
# Redeploying to a new version can cause this
if not os.path.exists(filename):
continue
with open(filename) as f:
content = unicode(f.read(), settings.DEFAULT_CHARSET)
results = parse_file(content, os.path.splitext(filename)[-1])
for text, plural, hint, group in results:
if not text:
logger.warn("Empty translation discovered: '{}', '{}', '{}', '{}'".format(text, plural, hint, group))
continue
with transaction.atomic(xg=True):
key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE)
try:
mt = MasterTranslation.objects.get(pk=key)
except MasterTranslation.DoesNotExist:
mt = MasterTranslation(
pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE
)
# By the very act of getting here, this is true
mt.used_in_code_or_templates = True
# If we last updated during this scan, then append, otherwise replace
if mt.last_updated_by_scan_uuid == scan_id:
mt.used_by_groups_in_code_or_templates.add(group)
else:
mt.used_by_groups_in_code_or_templates = { group }
mt.last_updated_by_scan_uuid = scan_id
mt.save()
with transaction.atomic():
marshall.refresh_from_db()
marshall.files_left_to_process -= len(filenames)
marshall.save()
def test_blocktrans_tag_and_noescape(self):
text = "\nTest blocktrans & unescaping\n"
content = """
{% load fluent %}
{% blocktrans noescape %}
Test blocktrans & unescaping
{% endblocktrans %}
"""
results = parse_file(content, ".html")
expected = [(text, '', '', DEFAULT_TRANSLATION_GROUP)]
self.assertEqual(results, expected)
rendered = Template(content).render(Context())
self.assertTrue(text in rendered)
# create master translation for the string and test it renders correctly
key = MasterTranslation.generate_key(text, '', 'en-us')
MasterTranslation.objects.create(
pk=key, text=text, language_code='en-us'
)
rendered = Template(content).render(Context())
self.assertTrue("Test blocktrans & unescaping" in rendered)
def _scan_list(marshall, scan_id, filenames):
""" Given a list of filenames (file paths), of templates and/or python files, scan them for
translatable strings and create corresponding MasterTranslation objects.
"""
# FIXME: Need to clean up the translations which aren't in use anymore
for filename in filenames:
# Redeploying to a new version can cause this
if not os.path.exists(filename):
continue
with open(filename) as f:
content = unicode(f.read(), settings.DEFAULT_CHARSET)
results = parse_file(content, os.path.splitext(filename)[-1])
for text, plural, hint, group in results:
if not text:
logger.warn("Empty translation discovered: '{}', '{}', '{}', '{}'".format(text, plural, hint, group))
continue
with transaction.atomic(xg=True):
key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE)
try:
mt = MasterTranslation.objects.get(pk=key)
except MasterTranslation.DoesNotExist:
mt = MasterTranslation(
pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE
)
# By the very act of getting here, this is true
mt.used_in_code_or_templates = True
# If we last updated during this scan, then append, otherwise replace
if mt.last_updated_by_scan_uuid == unicode(scan_id):
mt.used_by_groups_in_code_or_templates.add(group)
else:
mt.used_by_groups_in_code_or_templates = { group }
mt.last_updated_by_scan_uuid = scan_id
mt.save()
# Update the ScanMarshall object with the reduced number of `files_left_to_process`.
# Do this with several retries, so that if the transction collides with another task (which is
# quite likely) this whole task doesn't fail and retry (which would be fine but inefficient).
for retry in xrange(3):
try:
with transaction.atomic():
marshall.refresh_from_db()
marshall.files_left_to_process -= len(filenames)
marshall.save()
return
except TransactionFailedError:
msg = "Transaction failed trying to decrement 'files_left_to_process' on ScanMarshall, "
msg += ("retrying..." if retry < 2 else "giving up, task will error and retry.")
logger.info(msg)
if retry < 2:
# Back off by random number of ms. This helps prevent 2 colliding tasks from
# repeatedly re-colliding.
time.sleep(random.randint(0, 1000) / 1000.0)
# Tried 3 times, give up, let the task fail and retry
raise
def _scan_list(marshall, scan_id, filenames):
""" Given a list of filenames (file paths), of templates and/or python files, scan them for
translatable strings and create corresponding MasterTranslation objects.
"""
# FIXME: Need to clean up the translations which aren't in use anymore
for filename in filenames:
# Redeploying to a new version can cause this
if not os.path.exists(filename):
continue
with open(filename) as f:
content = unicode(f.read(), settings.DEFAULT_CHARSET)
results = parse_file(content, os.path.splitext(filename)[-1])
for text, plural, hint, group in results:
if not text:
logger.warn(
"Empty translation discovered: '{}', '{}', '{}', '{}'".
format(text, plural, hint, group))
continue
with transaction.atomic(xg=True):
key = MasterTranslation.generate_key(text, hint,
settings.LANGUAGE_CODE)
try:
mt = MasterTranslation.objects.get(pk=key)
except MasterTranslation.DoesNotExist:
mt = MasterTranslation(
pk=key,
text=text,
hint=hint,
language_code=settings.LANGUAGE_CODE)
# By the very act of getting here, this is true
mt.used_in_code_or_templates = True
# If we last updated during this scan, then append, otherwise replace
if mt.last_updated_by_scan_uuid == unicode(scan_id):
mt.used_by_groups_in_code_or_templates.add(group)
else:
mt.used_by_groups_in_code_or_templates = {group}
mt.last_updated_by_scan_uuid = scan_id
mt.save()
# Update the ScanMarshall object with the reduced number of `files_left_to_process`.
# Do this with several retries, so that if the transction collides with another task (which is
# quite likely) this whole task doesn't fail and retry (which would be fine but inefficient).
for retry in xrange(3):
try:
with transaction.atomic():
marshall.refresh_from_db()
marshall.files_left_to_process -= len(filenames)
marshall.save()
return
except TransactionFailedError:
msg = "Transaction failed trying to decrement 'files_left_to_process' on ScanMarshall, "
msg += ("retrying..."
if retry < 2 else "giving up, task will error and retry.")
logger.info(msg)
if retry < 2:
# Back off by random number of ms. This helps prevent 2 colliding tasks from
# repeatedly re-colliding.
time.sleep(random.randint(0, 1000) / 1000.0)
# Tried 3 times, give up, let the task fail and retry
raise
