def diff_resources(left_path, right_path):
parser = FluentParser(with_spans=False)
serializer = FluentSerializer(with_junk=True)
lines = []
for p in (left_path, right_path):
with codecs.open(p, encoding='utf-8') as fh:
res = parser.parse(fh.read())
lines.append(serializer.serialize(res).splitlines(True))
sys.stdout.writelines(
chunk for chunk in unified_diff(lines[0], lines[1], left_path, right_path)
)
def execute(self):
for keys in self.group:
full_message = FluentSerializedMessage.from_lokalise_keys(
self.group[keys])
parsed_message = FluentParser().parse(full_message)
ru_full_path = self.group[keys][0].get_file_path().ru
ru_file = FluentFile(ru_full_path)
try:
ru_file_parsed = ru_file.read_parsed_data()
except:
logging.error(f'Файла {ru_file.full_path} не существует')
continue
manager = LokaliseFluentAstComparerManager(
sourse_parsed=ru_file_parsed, target_parsed=parsed_message)
for_update = manager.for_update()
for_create = manager.for_create()
for_delete = manager.for_delete()
if len(for_update):
updated_ru_file_parsed = manager.update(for_update)
updated_ru_file_serialized = FluentSerializer(
with_junk=True).serialize(updated_ru_file_parsed)
ru_file.save_data(updated_ru_file_serialized)
updated_keys = list(
map(lambda el: el.get_id_name(), for_update))
logging.info(
f'Обновлены ключи: {updated_keys} в файле {ru_file.full_path}'
)
def to_serialized_message(string_message):
if not string_message:
return None
ast_message = FluentParser().parse(string_message)
serialized = FluentSerializer(with_junk=True).serialize(ast_message)
return serialized if serialized else ''
def migrate_ftl_translations_to_0_6_4(apps, schema):
"""
Converts all FTL translations to the latest (0.5) syntax and serializes
them with the latest (0.6.4) serializer to prevent creating duplicate
translations on save. Also convert corresponding TranslationMemoryEntries.
See bugs 1441020 and 1442201 for more details.
"""
parser = FluentParser()
serializer = FluentSerializer()
# Translations
Translation = apps.get_model("base", "Translation")
translations = Translation.objects.filter(entity__resource__format="ftl")
translations_to_update = []
for t in translations:
current = t.string
ast = parser.parse_entry(current)
t.string = serializer.serialize_entry(ast)
if t.string != current:
translations_to_update.append(t)
bulk_update(translations_to_update, update_fields=["string"])
# Translation Memory Entries
TranslationMemoryEntry = apps.get_model("base", "TranslationMemoryEntry")
updated_pks = [x.pk for x in translations_to_update]
tms = TranslationMemoryEntry.objects.filter(
translation__pk__in=updated_pks)
tms_to_update = []
for tm in tms:
current = tm.target
ast = parser.parse_entry(current)
tm.target = serializer.serialize_entry(ast)
if tm.target != current:
tms_to_update.append(tm)
bulk_update(tms_to_update, update_fields=["target"])
def migrate_ftl_translations_to_0_6_4(apps, schema):
"""
Converts all FTL translations to the latest (0.5) syntax and serializes
them with the latest (0.6.4) serializer to prevent creating duplicate
translations on save. Also convert corresponding TranslationMemoryEntries.
See bugs 1441020 and 1442201 for more details.
"""
parser = FluentParser()
serializer = FluentSerializer()
# Translations
Translation = apps.get_model('base', 'Translation')
translations = Translation.objects.filter(entity__resource__format='ftl')
translations_to_update = []
for t in translations:
current = t.string
ast = parser.parse_entry(current)
t.string = serializer.serialize_entry(ast)
if t.string != current:
translations_to_update.append(t)
bulk_update(translations_to_update, update_fields=['string'])
# Translation Memory Entries
TranslationMemoryEntry = apps.get_model('base', 'TranslationMemoryEntry')
updated_pks = [x.pk for x in translations_to_update]
tms = TranslationMemoryEntry.objects.filter(translation__pk__in=updated_pks)
tms_to_update = []
for tm in tms:
current = tm.target
ast = parser.parse_entry(current)
tm.target = serializer.serialize_entry(ast)
if tm.target != current:
tms_to_update.append(tm)
bulk_update(tms_to_update, update_fields=['target'])
def test_invalid_resource(self):
serializer = FluentSerializer()
with self.assertRaisesRegex(Exception, 'Unknown resource type'):
serializer.serialize(None)
with self.assertRaisesRegex(Exception, 'Unknown resource type'):
serializer.serialize(object())
def test_invalid_expression(self):
serializer = FluentSerializer()
with self.assertRaisesRegexp(Exception, 'Unknown expression type'):
serializer.serialize_expression(None)
with self.assertRaisesRegexp(Exception, 'Unknown expression type'):
serializer.serialize_expression(object())
from bleach.linkifier import Linker
from django_jinja import library
from fluent.syntax import FluentParser, FluentSerializer, ast
from fluent.syntax.serializer import serialize_expression
from django import template
from django.conf import settings
from django.contrib.humanize.templatetags import humanize
from django.contrib.staticfiles.storage import staticfiles_storage
from django.core.serializers.json import DjangoJSONEncoder
from django.urls import reverse
from django.utils.http import url_has_allowed_host_and_scheme
register = template.Library()
parser = FluentParser()
serializer = FluentSerializer()
@library.global_function
def url(viewname, *args, **kwargs):
"""Helper for Django's ``reverse`` in templates."""
return reverse(viewname, args=args, kwargs=kwargs)
@library.global_function
def return_url(request):
"""Get an url of the previous page."""
url = request.POST.get("return_url", request.META.get("HTTP_REFERER", "/"))
if not url_has_allowed_host_and_scheme(url, settings.ALLOWED_HOSTS):
return settings.SITE_URL
return url
def remove_comments_from_ftl_translations(apps, schema):
"""
Remove comments from the string column of FTL Translations and Entities,
and source & target columns of FTL TranslationMemoryEntries.
See bug 1501168 for more details.
"""
parser = FluentParser()
serializer = FluentSerializer()
# Translations
Translation = apps.get_model("base", "Translation")
translations = Translation.objects.filter(entity__resource__format="ftl")
translations_to_update = []
for t in translations:
current = t.string
ast = parser.parse_entry(current)
ast.comment = None
t.string = serializer.serialize_entry(ast)
if t.string != current:
translations_to_update.append(t)
bulk_update(translations_to_update, update_fields=["string"])
# Entities
Entity = apps.get_model("base", "Entity")
entities = Entity.objects.filter(resource__format="ftl")
entities_to_update = []
for e in entities:
current = e.string
ast = parser.parse_entry(current)
ast.comment = None
e.string = serializer.serialize_entry(ast)
if e.string != current:
entities_to_update.append(e)
bulk_update(entities_to_update, update_fields=["string"])
# Translation Memory Entries
TranslationMemoryEntry = apps.get_model("base", "TranslationMemoryEntry")
tmes = TranslationMemoryEntry.objects.filter(entity__resource__format="ftl")
tmes_to_update = []
for tme in tmes:
current_source = tme.source
ast = parser.parse_entry(current_source)
ast.comment = None
tme.source = serializer.serialize_entry(ast)
current_target = tme.target
ast = parser.parse_entry(current_target)
ast.comment = None
tme.target = serializer.serialize_entry(ast)
if tme.source != current_source or tme.target != current_target:
tmes_to_update.append(tme)
bulk_update(tmes_to_update, update_fields=["source", "target"])
def pretty_ftl(text):
parser = FluentParser()
serializer = FluentSerializer(with_junk=False)
res = parser.parse(dedent_ftl(text))
return serializer.serialize(res)
def serialize_data(self, parsed_file_data: ast.Resource):
from fluent.syntax import FluentSerializer
return FluentSerializer(with_junk=True).serialize(parsed_file_data)
def pretty_expr(text):
parser = FluentParser()
serializer = FluentSerializer(with_junk=False)
entry = parser.parse_entry(dedent_ftl(text))
expr = entry.value.elements[0].expression
return serializer.serialize_expression(expr)
def format_serialized_file_data(cls, file_data: typing.AnyStr):
parsed_data = FluentParser().parse(file_data)
return FluentSerializer(with_junk=True).serialize(parsed_data)
if not ru_element_id_name or not en_element_id_name:
return False
if ru_element_id_name == en_element_id_name:
return ru_message
else:
return None
######################################## Var definitions ###############################################################
logging.basicConfig(level=logging.INFO)
project = Project()
parser = FluentParser()
serializer = FluentSerializer(with_junk=True)
files_finder = FilesFinder(project)
key_finder = KeyFinder(files_finder.get_files_pars())
########################################################################################################################
print('Проверка актуальности файлов ...')
created_files = files_finder.execute()
if len(created_files):
print('Форматирование созданных файлов ...')
FluentFormatter.format(created_files)
print('Проверка актуальности ключей ...')
changed_files = key_finder.execute()
if len(changed_files):
print('Форматирование изменённых файлов ...')
FluentFormatter.format(changed_files)
