def test_load_str_po(self): """Test that a str with multibyte uft-8 chars can parsed correctly""" MasterTranslation(text=u"This — that", language_code="en").save() MasterTranslation(text=u"something something something something translate", language_code="en").save() pofile = """%s msgid "This — that" msgstr "Deise — dass" """ % POFILE errors = import_translations_from_po(pofile, "de", "en") self.assertEqual(errors, [])
def test_export(self): MasterTranslation( text=u"Wave", hint="Hand gesture", language_code="en").save() MasterTranslation( text=u"Wave", hint="Oceanic", language_code="en").save() MasterTranslation( text="One message removed", hint="Message(s) removed plural", plural_text="%d messages deleted", language_code="en").save() MasterTranslation( text=u"Something to translate", language_code="en").save() MasterTranslation( text=u"Product®™ — Special chars", language_code="en").save() po_file = polib.pofile(unicode( export_translations_to_po('en').content.decode('utf-8'))) self.assertEqual(EXPECTED_EXPORT_PO_FILE, po_file.__unicode__())
def test_export_handles_language_region_codes(self): # Fluent should handle 'en-US' as a language code. Previously # `export_translations_to_po('en-US')` would raise KeyError. MasterTranslation(text=u'Foo', hint=u'Bar', language_code='en').save() result = unicode(export_translations_to_po('en-US')) expected = ( u'Content-Type: text/plain\r\n' u'Content-Disposition: attachment; filename=django.po\r\n\r\n' u'#. Bar\n' u'msgctxt "Bar"\n' u'msgid "Foo"\n' u'msgstr "Foo"\n' ) self.assertEqual(result, expected)
def _scan_list(marshall, scan_id, filenames): # FIXME: Need to clean up the translations which aren't in use anymore for filename in filenames: # Redeploying to a new version can cause this if not os.path.exists(filename): continue with open(filename) as f: content = f.read() results = parse_file(content, os.path.splitext(filename)[-1]) for text, plural, hint, group in results: if not text: logger.warn("Empty translation discovered: '{}', '{}', '{}', '{}'".format(text, plural, hint, group)) continue with transaction.atomic(xg=True): key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE) try: mt = MasterTranslation.objects.get(pk=key) except MasterTranslation.DoesNotExist: mt = MasterTranslation( pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE ) # By the very act of getting here, this is true mt.used_in_code_or_templates = True # If we last updated during this scan, then append, otherwise replace if mt.last_updated_by_scan_uuid == scan_id: mt.used_by_groups_in_code_or_templates.add(group) else: mt.used_by_groups_in_code_or_templates = { group } mt.last_updated_by_scan_uuid = scan_id mt.save() with transaction.atomic(): marshall.refresh_from_db() marshall.files_left_to_process -= len(filenames) marshall.save()
def _scan_list(marshall, scan_id, filenames): """ Given a list of filenames (file paths), of templates and/or python files, scan them for translatable strings and create corresponding MasterTranslation objects. """ # FIXME: Need to clean up the translations which aren't in use anymore for filename in filenames: # Redeploying to a new version can cause this if not os.path.exists(filename): continue with open(filename) as f: content = unicode(f.read(), settings.DEFAULT_CHARSET) results = parse_file(content, os.path.splitext(filename)[-1]) for text, plural, hint, group in results: if not text: logger.warn( "Empty translation discovered: '{}', '{}', '{}', '{}'". format(text, plural, hint, group)) continue with transaction.atomic(xg=True): key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE) try: mt = MasterTranslation.objects.get(pk=key) except MasterTranslation.DoesNotExist: mt = MasterTranslation( pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE) # By the very act of getting here, this is true mt.used_in_code_or_templates = True # If we last updated during this scan, then append, otherwise replace if mt.last_updated_by_scan_uuid == unicode(scan_id): mt.used_by_groups_in_code_or_templates.add(group) else: mt.used_by_groups_in_code_or_templates = {group} mt.last_updated_by_scan_uuid = scan_id mt.save() # Update the ScanMarshall object with the reduced number of `files_left_to_process`. # Do this with several retries, so that if the transction collides with another task (which is # quite likely) this whole task doesn't fail and retry (which would be fine but inefficient). for retry in xrange(3): try: with transaction.atomic(): marshall.refresh_from_db() marshall.files_left_to_process -= len(filenames) marshall.save() return except TransactionFailedError: msg = "Transaction failed trying to decrement 'files_left_to_process' on ScanMarshall, " msg += ("retrying..." if retry < 2 else "giving up, task will error and retry.") logger.info(msg) if retry < 2: # Back off by random number of ms. This helps prevent 2 colliding tasks from # repeatedly re-colliding. time.sleep(random.randint(0, 1000) / 1000.0) # Tried 3 times, give up, let the task fail and retry raise