def test_pk_is_set_correctly(self): mt1 = MasterTranslation.objects.create(text="Hello World!", language_code="en") self.assertEqual(mt1.pk, MasterTranslation.generate_key("Hello World!", "", "en")) mt2 = MasterTranslation.objects.create(text="Hello World!", language_code="fr") self.assertEqual(mt2.pk, MasterTranslation.generate_key("Hello World!", "", "fr")) # Make sure that it differs self.assertNotEqual(mt1.pk, mt2.pk)
def test_pk_is_set_correctly(self): mt1 = MasterTranslation.objects.create(text="Hello World!", language_code="en") self.assertEqual( mt1.pk, MasterTranslation.generate_key("Hello World!", "", "en")) mt2 = MasterTranslation.objects.create(text="Hello World!", language_code="fr") self.assertEqual( mt2.pk, MasterTranslation.generate_key("Hello World!", "", "fr")) # Make sure that it differs self.assertNotEqual(mt1.pk, mt2.pk)
def test_blocktrans_tag_with_variable_escaping(self): text = '\n<a href="http://google.com">%(name)s</a> without group\n' content = """ {% load fluent %} {% blocktrans %} <a href="http://google.com">{{ name }}</a> without group {% endblocktrans %} """ results = parse_file(content, ".html") expected = [ (text, '', '', DEFAULT_TRANSLATION_GROUP), ] self.assertEqual(results, expected) data = {'name': "Ola & Ola"} rendered = Template(content).render(Context(data)) self.assertTrue('<a href="http://google.com">Ola & Ola</a> without group' in rendered) # create master translation for the string and test it renders correctly key = MasterTranslation.generate_key(text, '', 'en-us') MasterTranslation.objects.create( pk=key, text=text, language_code='en-us' ) rendered = Template(content).render(Context(data)) self.assertTrue('<a href="http://google.com">Ola & Ola</a> without group' in rendered)
def test_blocktrans_tag_with_variable_and_group_escaping(self): text = '\n<a href="http://google.com">%(name)s</a> in group\n' content = """ {% load fluent %} {% blocktrans group "public" %} <a href="http://google.com">{{ name }}</a> in group {% endblocktrans %} """ results = parse_file(content, ".html") expected = [ (text, '', '', 'public'), ] self.assertEqual(results, expected) data = {'name': "Ola & Ola"} rendered = Template(content).render(Context(data)) self.assertTrue('<a href="http://google.com">Ola & Ola</a> in group' in rendered) # create master translation for the string and test it renders correctly key = MasterTranslation.generate_key(text, '', 'en-us') mt = MasterTranslation.objects.create( pk=key, text=text, language_code='en-us' ) mt.used_by_groups_in_code_or_templates = {'public'} mt.save() rendered = Template(content).render(Context(data)) self.assertTrue('<a href="http://google.com">Ola & Ola</a> in group' in rendered)
def test_blocktrans_tag_with_group(self): text = "\nTest trans block with group\n" content = """ {% load fluent %} {% blocktrans group "public" %} Test trans block with group {% endblocktrans %} """ results = parse_file(content, ".html") expected = [ (text, '', '', 'public'), ] self.assertEqual(results, expected) rendered = Template(content).render(Context()) self.assertTrue(text in rendered) # create master translation for the string and test it renders correctly key = MasterTranslation.generate_key(text, '', 'en-us') mt = MasterTranslation.objects.create( pk=key, text=text, language_code='en-us' ) mt.used_by_groups_in_code_or_templates = {'public'} mt.save() rendered = Template(content).render(Context()) self.assertTrue("Test trans block with group" in rendered)
def _scan_list(marshall, scan_id, filenames): # FIXME: Need to clean up the translations which aren't in use anymore for filename in filenames: # Redeploying to a new version can cause this if not os.path.exists(filename): continue with open(filename) as f: content = unicode(f.read(), settings.DEFAULT_CHARSET) results = parse_file(content, os.path.splitext(filename)[-1]) for text, plural, hint, group in results: if not text: logger.warn("Empty translation discovered: '{}', '{}', '{}', '{}'".format(text, plural, hint, group)) continue with transaction.atomic(xg=True): key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE) try: mt = MasterTranslation.objects.get(pk=key) except MasterTranslation.DoesNotExist: mt = MasterTranslation( pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE ) # By the very act of getting here, this is true mt.used_in_code_or_templates = True # If we last updated during this scan, then append, otherwise replace if mt.last_updated_by_scan_uuid == scan_id: mt.used_by_groups_in_code_or_templates.add(group) else: mt.used_by_groups_in_code_or_templates = { group } mt.last_updated_by_scan_uuid = scan_id mt.save() with transaction.atomic(): marshall.refresh_from_db() marshall.files_left_to_process -= len(filenames) marshall.save()
def _scan_list(marshall, scan_id, filenames): # FIXME: Need to clean up the translations which aren't in use anymore for filename in filenames: # Redeploying to a new version can cause this if not os.path.exists(filename): continue with open(filename) as f: content = f.read() results = parse_file(content, os.path.splitext(filename)[-1]) for text, plural, hint, group in results: if not text: logger.warn("Empty translation discovered: '{}', '{}', '{}', '{}'".format(text, plural, hint, group)) continue with transaction.atomic(xg=True): key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE) try: mt = MasterTranslation.objects.get(pk=key) except MasterTranslation.DoesNotExist: mt = MasterTranslation( pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE ) # By the very act of getting here, this is true mt.used_in_code_or_templates = True # If we last updated during this scan, then append, otherwise replace if mt.last_updated_by_scan_uuid == scan_id: mt.used_by_groups_in_code_or_templates.add(group) else: mt.used_by_groups_in_code_or_templates = { group } mt.last_updated_by_scan_uuid = scan_id mt.save() with transaction.atomic(): marshall.refresh_from_db() marshall.files_left_to_process -= len(filenames) marshall.save()
def test_trans_tag_without_group(self): text = "Test trans string without group" content = """ {% load fluent %} {% trans "Test trans string without group" %} """ results = parse_file(content, ".html") expected = [(text, '', '', DEFAULT_TRANSLATION_GROUP)] self.assertEqual(results, expected) rendered = Template(content).render(Context()) self.assertTrue(text in rendered) # create master translation for the string and test it renders correctly key = MasterTranslation.generate_key(text, '', 'en-us') MasterTranslation.objects.create( pk=key, text=text, language_code='en-us' ) rendered = Template(content).render(Context()) self.assertTrue("Test trans string without group" in rendered)
def test_blocktrans_tag_and_noescape(self): text = "\nTest blocktrans & unescaping\n" content = """ {% load fluent %} {% blocktrans noescape %} Test blocktrans & unescaping {% endblocktrans %} """ results = parse_file(content, ".html") expected = [(text, '', '', DEFAULT_TRANSLATION_GROUP)] self.assertEqual(results, expected) rendered = Template(content).render(Context()) self.assertTrue(text in rendered) # create master translation for the string and test it renders correctly key = MasterTranslation.generate_key(text, '', 'en-us') MasterTranslation.objects.create( pk=key, text=text, language_code='en-us' ) rendered = Template(content).render(Context()) self.assertTrue("Test blocktrans & unescaping" in rendered)
def _scan_list(marshall, scan_id, filenames): """ Given a list of filenames (file paths), of templates and/or python files, scan them for translatable strings and create corresponding MasterTranslation objects. """ # FIXME: Need to clean up the translations which aren't in use anymore for filename in filenames: # Redeploying to a new version can cause this if not os.path.exists(filename): continue with open(filename) as f: content = unicode(f.read(), settings.DEFAULT_CHARSET) results = parse_file(content, os.path.splitext(filename)[-1]) for text, plural, hint, group in results: if not text: logger.warn("Empty translation discovered: '{}', '{}', '{}', '{}'".format(text, plural, hint, group)) continue with transaction.atomic(xg=True): key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE) try: mt = MasterTranslation.objects.get(pk=key) except MasterTranslation.DoesNotExist: mt = MasterTranslation( pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE ) # By the very act of getting here, this is true mt.used_in_code_or_templates = True # If we last updated during this scan, then append, otherwise replace if mt.last_updated_by_scan_uuid == unicode(scan_id): mt.used_by_groups_in_code_or_templates.add(group) else: mt.used_by_groups_in_code_or_templates = { group } mt.last_updated_by_scan_uuid = scan_id mt.save() # Update the ScanMarshall object with the reduced number of `files_left_to_process`. # Do this with several retries, so that if the transction collides with another task (which is # quite likely) this whole task doesn't fail and retry (which would be fine but inefficient). for retry in xrange(3): try: with transaction.atomic(): marshall.refresh_from_db() marshall.files_left_to_process -= len(filenames) marshall.save() return except TransactionFailedError: msg = "Transaction failed trying to decrement 'files_left_to_process' on ScanMarshall, " msg += ("retrying..." if retry < 2 else "giving up, task will error and retry.") logger.info(msg) if retry < 2: # Back off by random number of ms. This helps prevent 2 colliding tasks from # repeatedly re-colliding. time.sleep(random.randint(0, 1000) / 1000.0) # Tried 3 times, give up, let the task fail and retry raise
def _scan_list(marshall, scan_id, filenames): """ Given a list of filenames (file paths), of templates and/or python files, scan them for translatable strings and create corresponding MasterTranslation objects. """ # FIXME: Need to clean up the translations which aren't in use anymore for filename in filenames: # Redeploying to a new version can cause this if not os.path.exists(filename): continue with open(filename) as f: content = unicode(f.read(), settings.DEFAULT_CHARSET) results = parse_file(content, os.path.splitext(filename)[-1]) for text, plural, hint, group in results: if not text: logger.warn( "Empty translation discovered: '{}', '{}', '{}', '{}'". format(text, plural, hint, group)) continue with transaction.atomic(xg=True): key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE) try: mt = MasterTranslation.objects.get(pk=key) except MasterTranslation.DoesNotExist: mt = MasterTranslation( pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE) # By the very act of getting here, this is true mt.used_in_code_or_templates = True # If we last updated during this scan, then append, otherwise replace if mt.last_updated_by_scan_uuid == unicode(scan_id): mt.used_by_groups_in_code_or_templates.add(group) else: mt.used_by_groups_in_code_or_templates = {group} mt.last_updated_by_scan_uuid = scan_id mt.save() # Update the ScanMarshall object with the reduced number of `files_left_to_process`. # Do this with several retries, so that if the transction collides with another task (which is # quite likely) this whole task doesn't fail and retry (which would be fine but inefficient). for retry in xrange(3): try: with transaction.atomic(): marshall.refresh_from_db() marshall.files_left_to_process -= len(filenames) marshall.save() return except TransactionFailedError: msg = "Transaction failed trying to decrement 'files_left_to_process' on ScanMarshall, " msg += ("retrying..." if retry < 2 else "giving up, task will error and retry.") logger.info(msg) if retry < 2: # Back off by random number of ms. This helps prevent 2 colliding tasks from # repeatedly re-colliding. time.sleep(random.randint(0, 1000) / 1000.0) # Tried 3 times, give up, let the task fail and retry raise