def __init__(self, lang, reference_dir, localization_dir):
self.fluent_parser = FluentParser(with_spans=False)
self.fluent_serializer = FluentSerializer()
# An iterable of plural category names relevant to the context's
# language. E.g. ('one', 'other') for English.
try:
self.plural_categories = CATEGORIES_BY_LOCALE[lang]
except KeyError as locale_key:
logger = logging.getLogger('migrate')
logger.warning('Plural rule for "{}" is not defined in '
'compare-locales'.format(locale_key))
self.plural_categories = ('one', 'other')
# Paths to directories with input data, relative to CWD.
self.reference_dir = reference_dir
self.localization_dir = localization_dir
# Parsed input resources stored by resource path.
self.reference_resources = {}
self.localization_resources = {}
self.target_resources = {}
# An iterable of `FTL.Message` objects some of whose nodes can be the
# transform operations.
self.transforms = {}
# A dict whose keys are `(path, key)` tuples corresponding to target
# FTL translations, and values are sets of `(path, key)` tuples
# corresponding to localized entities which will be migrated.
self.dependencies = {}
def __init__(
self, lang, reference_dir, localization_dir, enforce_translated=False
):
self.fluent_parser = FluentParser(with_spans=False)
self.fluent_serializer = FluentSerializer()
# An iterable of plural category names relevant to the context's
# language. E.g. ('one', 'other') for English.
self.plural_categories = get_plural(lang)
if self.plural_categories is None:
logger = logging.getLogger('migrate')
logger.warning(
'Plural rule for "{}" is not defined in '
'compare-locales'.format(lang))
self.plural_categories = ('one', 'other')
self.enforce_translated = enforce_translated
# Parsed input resources stored by resource path.
self.reference_resources = {}
self.localization_resources = {}
self.target_resources = {}
# An iterable of `FTL.Message` objects some of whose nodes can be the
# transform operations.
self.transforms = {}
# The evaluator instance is an AST transformer capable of walking an
# AST hierarchy and evaluating nodes which are migration Transforms.
self.evaluator = Evaluator(self)
class TestASTJSON(unittest.TestCase):
maxDiff = None
def setUp(self):
self.parser = FluentParser()
def test_simple_resource(self):
input = """\
foo = Foo
"""
ast1 = self.parser.parse(dedent_ftl(input))
json1 = ast1.to_json()
ast2 = from_json(json1)
json2 = ast2.to_json()
self.assertEqual(json1, json2)
def test_complex_resource(self):
input = """\
### A Resource comment
# A comment about shared-photos
shared-photos =
{ $user_name } { $photo_count ->
[0] hasn't added any photos yet
[one] added a new photo
*[other] added { $photo_count } new photos
}.
## A Section comment
// A Syntax 0.4 comment about liked-comment
liked-comment =
{ $user_name } liked your comment on { $user_gender ->
[male] his
[female] her
*[other] their
} post.
"""
ast1 = self.parser.parse(dedent_ftl(input))
json1 = ast1.to_json()
ast2 = from_json(json1)
json2 = ast2.to_json()
self.assertEqual(json1, json2)
def test_syntax_error(self):
input = """\
foo = Foo {
"""
ast1 = self.parser.parse(dedent_ftl(input))
json1 = ast1.to_json()
ast2 = from_json(json1)
json2 = ast2.to_json()
self.assertEqual(json1, json2)
def __init__(self, lang, reference_dir, localization_dir):
self.fluent_parser = FluentParser(with_spans=False)
self.fluent_serializer = FluentSerializer()
# An iterable of plural category names relevant to the context's
# language. E.g. ('one', 'other') for English.
try:
self.plural_categories = get_plural_categories(lang)
except RuntimeError as e:
logging.getLogger('migrate').warn(e)
self.plural_categories = get_plural_categories('en')
# Paths to directories with input data, relative to CWD.
self.reference_dir = reference_dir
self.localization_dir = localization_dir
# Parsed input resources stored by resource path.
self.reference_resources = {}
self.localization_resources = {}
# An iterable of `FTL.Message` objects some of whose nodes can be the
# transform operations.
self.transforms = {}
# A dict whose keys are `(path, key)` tuples corresponding to target
# FTL translations, and values are sets of `(path, key)` tuples
# corresponding to localized entities which will be migrated.
self.dependencies = {}
def test_differ_with_spans(self):
parser = FluentParser()
strings = [
("foo = Foo", "foo = Foo"),
("foo = { $arg }", "foo = { $arg }"),
]
messages = [(parser.parse_entry(a), parser.parse_entry(b))
for a, b in strings]
for a, b in messages:
self.assertFalse(a.equals(b, ignored_fields=None))
def test_default_behavior(self):
parser = FluentParser()
strings = [
("foo = Foo", "foo = Foo"),
("foo = Foo", "foo = Foo"),
("foo = { $arg }", "foo = { $arg }"),
]
messages = [(parser.parse_entry(a), parser.parse_entry(b))
for a, b in strings]
for a, b in messages:
self.assertTrue(a.equals(b))
def test_parser_without_spans(self):
parser = FluentParser(with_spans=False)
strings = [
("foo = Foo", "foo = Foo"),
("foo = Foo", "foo = Foo"),
("foo = { $arg }", "foo = { $arg }"),
]
messages = [(parser.parse_entry(a), parser.parse_entry(b))
for a, b in strings]
for a, b in messages:
self.assertTrue(a.equals(b))
def __init__(self, lang, reference_dir, localization_dir):
self.fluent_parser = FluentParser(with_spans=False)
self.fluent_serializer = FluentSerializer()
# An iterable of plural category names relevant to the context's
# language. E.g. ('one', 'other') for English.
try:
self.plural_categories = get_plural_categories(lang)
except RuntimeError as e:
print(e.message)
self.plural_categories = 'en'
# Paths to directories with input data, relative to CWD.
self.reference_dir = reference_dir
self.localization_dir = localization_dir
# Parsed input resources stored by resource path.
self.reference_resources = {}
self.localization_resources = {}
# An iterable of `FTL.Message` objects some of whose nodes can be the
# transform operations.
self.transforms = {}
# A dict whose keys are `(path, key)` tuples corresponding to target
# FTL translations, and values are sets of `(path, key)` tuples
# corresponding to localized entities which will be migrated.
self.dependencies = {}
def test(self):
resource = FluentParser().parse(
dedent_ftl('''\
one = Message
two = Messages
three = Has a
.an = Message string in the Attribute
'''))
prior_res_id = id(resource)
prior_msg_id = id(resource.body[1].value)
backup = resource.clone()
transformed = ReplaceTransformer('Message', 'Term').visit(resource)
self.assertEqual(prior_res_id, id(transformed))
self.assertEqual(prior_msg_id, id(transformed.body[1].value))
self.assertFalse(transformed.equals(backup))
self.assertEqual(transformed.body[1].value.elements[0].value, 'Terms')
class TestOrderEquals(unittest.TestCase):
def setUp(self):
self.parser = FluentParser()
def parse_ftl_entry(self, string):
return self.parser.parse_entry(dedent_ftl(string))
def test_attributes(self):
message1 = self.parse_ftl_entry("""\
foo
.attr1 = Attr1
.attr2 = Attr2
""")
message2 = self.parse_ftl_entry("""\
foo
.attr2 = Attr2
.attr1 = Attr1
""")
self.assertTrue(message1.equals(message2))
self.assertTrue(message2.equals(message1))
def test_variants(self):
message1 = self.parse_ftl_entry("""\
foo =
{ $num ->
[a] A
*[b] B
}
""")
message2 = self.parse_ftl_entry("""\
foo =
{ $num ->
*[b] B
[a] A
}
""")
self.assertTrue(message1.equals(message2))
self.assertTrue(message2.equals(message1))
def test_variants_with_numbers(self):
message1 = self.parse_ftl_entry("""\
foo =
{ $num ->
[1] A
*[b] B
}
""")
message2 = self.parse_ftl_entry("""\
foo =
{ $num ->
*[b] B
[1] A
}
""")
self.assertTrue(message1.equals(message2))
self.assertTrue(message2.equals(message1))
def test_resource(self):
resource = FluentParser().parse(
dedent_ftl('''\
one = Message
# Comment
two = Messages
three = Messages with
.an = Attribute
'''))
mv = MockVisitor()
mv.visit(resource)
self.assertEqual(mv.pattern_calls, 4)
self.assertDictEqual(
mv.calls, {
'Resource': 1,
'Comment': 1,
'Message': 3,
'Identifier': 4,
'Attribute': 1,
'Span': 10,
})
class TestIgnoredFields(unittest.TestCase):
def setUp(self):
self.parser = FluentParser()
def parse_ftl_entry(self, string):
return self.parser.parse_entry(dedent_ftl(string))
def test_ignore_value(self):
a = self.parse_ftl_entry("foo = Foo")
b = self.parse_ftl_entry("foo = Bar")
self.assertTrue(a.equals(b, ignored_fields=['value']))
def test_ignore_value_span(self):
a = self.parse_ftl_entry("foo = Foo")
b = self.parse_ftl_entry("foo = Foobar")
self.assertTrue(a.equals(b, ignored_fields=['span', 'value']))
self.assertFalse(a.equals(b, ignored_fields=['value']))
def test_ignore_comments(self):
a = self.parse_ftl_entry("""\
# Comment A
foo = Foo
""")
b = self.parse_ftl_entry("""\
# Comment B
foo = Foo
""")
c = self.parse_ftl_entry("""\
# Comment CC
foo = Foo
""")
self.assertTrue(a.equals(b, ignored_fields=['comment']))
self.assertFalse(a.equals(c, ignored_fields=['comment']))
self.assertTrue(a.equals(c, ignored_fields=['comment', 'span']))
class MergeContext(object):
"""Stateful context for merging translation resources.
`MergeContext` must be configured with the target language and the
directory locations of the input data.
The transformation takes four types of input data:
- The en-US FTL reference files which will be used as templates for
message order, comments and sections.
- The current FTL files for the given language.
- The legacy (DTD, properties) translation files for the given
language. The translations from these files will be transformed
into FTL and merged into the existing FTL files for this language.
- A list of `FTL.Message` objects some of whose nodes are special
helper or transform nodes:
helpers: EXTERNAL_ARGUMENT, MESSAGE_REFERENCE
transforms: COPY, REPLACE_IN_TEXT, REPLACE, PLURALS, CONCAT
"""
def __init__(self, lang, reference_dir, localization_dir):
self.fluent_parser = FluentParser(with_spans=False)
self.fluent_serializer = FluentSerializer()
# An iterable of plural category names relevant to the context's
# language. E.g. ('one', 'other') for English.
try:
self.plural_categories = get_plural_categories(lang)
except RuntimeError as e:
logging.getLogger('migrate').warn(e)
self.plural_categories = get_plural_categories('en')
# Paths to directories with input data, relative to CWD.
self.reference_dir = reference_dir
self.localization_dir = localization_dir
# Parsed input resources stored by resource path.
self.reference_resources = {}
self.localization_resources = {}
# An iterable of `FTL.Message` objects some of whose nodes can be the
# transform operations.
self.transforms = {}
# A dict whose keys are `(path, key)` tuples corresponding to target
# FTL translations, and values are sets of `(path, key)` tuples
# corresponding to localized entities which will be migrated.
self.dependencies = {}
def read_ftl_resource(self, path):
"""Read an FTL resource and parse it into an AST."""
f = codecs.open(path, 'r', 'utf8')
try:
contents = f.read()
except UnicodeDecodeError as err:
logger = logging.getLogger('migrate')
logger.warn('Unable to read file {}: {}'.format(path, err))
raise err
finally:
f.close()
ast = self.fluent_parser.parse(contents)
annots = [annot for entry in ast.body for annot in entry.annotations]
if len(annots):
logger = logging.getLogger('migrate')
for annot in annots:
msg = annot.message
logger.warn('Syntax error in {}: {}'.format(path, msg))
return ast
def read_legacy_resource(self, path):
"""Read a legacy resource and parse it into a dict."""
parser = getParser(path)
parser.readFile(path)
# Transform the parsed result which is an iterator into a dict.
return {entity.key: entity.val for entity in parser}
def read_reference_ftl(self, path):
"""Read and parse a reference FTL file.
A missing resource file is a fatal error and will raise an
UnreadableReferenceError.
"""
fullpath = os.path.join(self.reference_dir, path)
try:
return self.read_ftl_resource(fullpath)
except IOError as err:
error_message = 'Missing reference file: {}'.format(fullpath)
logging.getLogger('migrate').error(error_message)
raise UnreadableReferenceError(error_message)
except UnicodeDecodeError as err:
error_message = 'Error reading file {}: {}'.format(fullpath, err)
logging.getLogger('migrate').error(error_message)
raise UnreadableReferenceError(error_message)
def read_localization_ftl(self, path):
"""Read and parse an existing localization FTL file.
Create a new FTL.Resource if the file doesn't exist or can't be
decoded.
"""
fullpath = os.path.join(self.localization_dir, path)
try:
return self.read_ftl_resource(fullpath)
except IOError:
logger = logging.getLogger('migrate')
logger.info('Localization file {} does not exist and '
'it will be created'.format(path))
return FTL.Resource()
except UnicodeDecodeError:
logger = logging.getLogger('migrate')
logger.warn('Localization file {} has broken encoding. '
'It will be re-created and some translations '
'may be lost'.format(path))
return FTL.Resource()
def maybe_add_localization(self, path):
"""Add a localization resource to migrate translations from.
Only legacy resources can be added as migration sources. The resource
may be missing on disk.
Uses a compare-locales parser to create a dict of (key, string value)
tuples.
"""
if path.endswith('.ftl'):
error_message = (
'Migrating translations from Fluent files is not supported '
'({})'.format(path))
logging.getLogger('migrate').error(error_message)
raise NotSupportedError(error_message)
try:
fullpath = os.path.join(self.localization_dir, path)
collection = self.read_legacy_resource(fullpath)
except IOError:
logger = logging.getLogger('migrate')
logger.warn('Missing localization file: {}'.format(path))
else:
self.localization_resources[path] = collection
def add_transforms(self, target, reference, transforms):
"""Define transforms for target using reference as template.
`target` is a path of the destination FTL file relative to the
localization directory. `reference` is a path to the template FTL
file relative to the reference directory.
Each transform is an extended FTL node with `Transform` nodes as some
values. Transforms are stored in their lazy AST form until
`merge_changeset` is called, at which point they are evaluated to real
FTL nodes with migrated translations.
Each transform is scanned for `Source` nodes which will be used to
build the list of dependencies for the transformed message.
"""
def get_sources(acc, cur):
if isinstance(cur, Source):
acc.add((cur.path, cur.key))
return acc
reference_ast = self.read_reference_ftl(reference)
self.reference_resources[target] = reference_ast
for node in transforms:
ident = node.id.name
# Scan `node` for `Source` nodes and collect the information they
# store into a set of dependencies.
dependencies = fold(get_sources, node, set())
# Set these sources as dependencies for the current transform.
self.dependencies[(target, ident)] = dependencies
# The target Fluent message should exist in the reference file. If
# it doesn't, it's probably a typo.
if get_message(reference_ast.body, ident) is None:
logger = logging.getLogger('migrate')
logger.warn('{} "{}" was not found in {}'.format(
type(node).__name__, ident, reference))
# Keep track of localization resource paths which were defined as
# sources in the transforms.
expected_paths = set()
# Read all legacy translation files defined in Source transforms. This
# may fail but a single missing legacy resource doesn't mean that the
# migration can't succeed.
for dependencies in self.dependencies.values():
for path in set(path for path, _ in dependencies):
expected_paths.add(path)
self.maybe_add_localization(path)
# However, if all legacy resources are missing, bail out early. There
# are no translations to migrate. We'd also get errors in hg annotate.
if len(expected_paths) > 0 and len(self.localization_resources) == 0:
error_message = 'No localization files were found'
logging.getLogger('migrate').error(error_message)
raise EmptyLocalizationError(error_message)
# Add the current transforms to any other transforms added earlier for
# this path.
path_transforms = self.transforms.setdefault(target, [])
path_transforms += transforms
if target not in self.localization_resources:
target_ast = self.read_localization_ftl(target)
self.localization_resources[target] = target_ast
def get_source(self, path, key):
"""Get an entity value from a localized legacy source.
Used by the `Source` transform.
"""
resource = self.localization_resources[path]
return resource.get(key, None)
def messages_equal(self, res1, res2):
"""Compare messages of two FTL resources.
Uses FTL.BaseNode.equals to compare all messages in two FTL resources.
If the order or number of messages differ, the result is also False.
"""
def message_id(message):
"Return the message's identifer name for sorting purposes."
return message.id.name
messages1 = sorted(
(entry for entry in res1.body if isinstance(entry, FTL.Message)),
key=message_id)
messages2 = sorted(
(entry for entry in res2.body if isinstance(entry, FTL.Message)),
key=message_id)
for msg1, msg2 in zip_longest(messages1, messages2):
if msg1 is None or msg2 is None:
return False
if not msg1.equals(msg2):
return False
return True
def merge_changeset(self, changeset=None):
"""Return a generator of FTL ASTs for the changeset.
The input data must be configured earlier using the `add_*` methods.
if given, `changeset` must be a set of (path, key) tuples describing
which legacy translations are to be merged. If `changeset` is None,
all legacy translations will be allowed to be migrated in a single
changeset.
The inner `in_changeset` function is used to determine if a message
should be migrated for the given changeset. It compares the legacy
dependencies of the transform defined for the message with legacy
translations available in the changeset. If all dependencies are
present, the message will be migrated.
Given `changeset`, return a dict whose keys are resource paths and
values are `FTL.Resource` instances. The values will also be used to
update this context's existing localization resources.
"""
if changeset is None:
# Merge all known legacy translations. Used in tests.
changeset = {
(path, key)
for path, strings in self.localization_resources.iteritems()
if not path.endswith('.ftl') for key in strings.iterkeys()
}
for path, reference in self.reference_resources.iteritems():
current = self.localization_resources[path]
transforms = self.transforms.get(path, [])
def in_changeset(ident):
"""Check if a message should be migrated.
A message will be migrated only if all of its dependencies
are present in the currently processed changeset.
If a transform defined for this message points to a missing
legacy translation, this message will not be merged. The
missing legacy dependency won't be present in the changeset.
This also means that partially translated messages (e.g.
constructed from two legacy strings out of which only one is
avaiable) will never be migrated.
"""
message_deps = self.dependencies.get((path, ident), None)
# Don't merge if we don't have a transform for this message.
if message_deps is None:
return False
# As a special case, if a transform exists but has no
# dependecies, it's a hardcoded `FTL.Node` which doesn't
# migrate any existing translation but rather creates a new
# one. Merge it.
if len(message_deps) == 0:
return True
# Make sure all the dependencies are present in the current
# changeset. Partial migrations are not currently supported.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=1321271
available_deps = message_deps & changeset
return message_deps == available_deps
# Merge legacy translations with the existing ones using the
# reference as a template.
snapshot = merge_resource(self, reference, current, transforms,
in_changeset)
# Skip this path if the messages in the merged snapshot are
# identical to those in the current state of the localization file.
# This may happen when:
#
# - none of the transforms is in the changset, or
# - all messages which would be migrated by the context's
# transforms already exist in the current state.
if self.messages_equal(current, snapshot):
continue
# Store the merged snapshot on the context so that the next merge
# already takes it into account as the existing localization.
self.localization_resources[path] = snapshot
# The result for this path is a complete `FTL.Resource`.
yield path, snapshot
def serialize_changeset(self, changeset):
"""Return a dict of serialized FTLs for the changeset.
Given `changeset`, return a dict whose keys are resource paths and
values are serialized FTL snapshots.
"""
return {
path: self.fluent_serializer.serialize(snapshot)
for path, snapshot in self.merge_changeset(changeset)
}
def setUp(self):
parser = FluentParser()
workload = os.path.join(os.path.dirname(__file__), 'fixtures_perf',
'workload-low.ftl')
with codecs.open(workload, encoding='utf-8') as f:
self.resource = parser.parse(f.read())
# coding=utf8
from __future__ import unicode_literals
from __future__ import absolute_import
import textwrap
import fluent.syntax.ast as FTL
from fluent.syntax.parser import FluentParser, FluentParserStream
fluent_parser = FluentParser(with_spans=False)
def parse(Parser, string):
if Parser is FluentParser:
return fluent_parser.parse(string)
# Parsing a legacy resource.
# Parse the string into the internal Context.
parser = Parser()
# compare-locales expects ASCII strings.
parser.readContents(string.encode('utf8'))
# Transform the parsed result which is an iterator into a dict.
return {ent.key: ent for ent in parser}
def ftl_resource_to_ast(code):
return fluent_parser.parse(ftl(code))
def ftl_resource_to_json(code):
class TestEntryEqualToSelf(unittest.TestCase):
def setUp(self):
self.parser = FluentParser()
def parse_ftl_entry(self, string):
return self.parser.parse_entry(dedent_ftl(string))
def test_same_simple_message(self):
message1 = self.parse_ftl_entry("""\
foo = Foo
""")
self.assertTrue(message1.equals(message1))
self.assertTrue(message1.equals(message1.clone()))
def test_same_selector_message(self):
message1 = self.parse_ftl_entry("""\
foo =
{ $num ->
[one] One
[two] Two
[few] Few
[many] Many
*[other] Other
}
""")
self.assertTrue(message1.equals(message1))
self.assertTrue(message1.equals(message1.clone()))
def test_same_complex_placeable_message(self):
message1 = self.parse_ftl_entry("""\
foo = Foo { NUMBER($num, style: "decimal") } Bar
""")
self.assertTrue(message1.equals(message1))
self.assertTrue(message1.equals(message1.clone()))
def test_same_message_with_attribute(self):
message1 = self.parse_ftl_entry("""\
foo =
.attr = Attr
""")
self.assertTrue(message1.equals(message1))
self.assertTrue(message1.equals(message1.clone()))
def test_same_message_with_attributes(self):
message1 = self.parse_ftl_entry("""\
foo =
.attr1 = Attr 1
.attr2 = Attr 2
""")
self.assertTrue(message1.equals(message1))
self.assertTrue(message1.equals(message1.clone()))
def test_same_junk(self):
message1 = self.parse_ftl_entry("""\
foo = Foo {
""")
self.assertTrue(message1.equals(message1))
self.assertTrue(message1.equals(message1.clone()))
class InternalContext(object):
"""Internal context for merging translation resources.
For the public interface, see `context.MigrationContext`.
"""
def __init__(
self, lang, reference_dir, localization_dir, enforce_translated=False
):
self.fluent_parser = FluentParser(with_spans=False)
self.fluent_serializer = FluentSerializer()
# An iterable of plural category names relevant to the context's
# language. E.g. ('one', 'other') for English.
self.plural_categories = get_plural(lang)
if self.plural_categories is None:
logger = logging.getLogger('migrate')
logger.warning(
'Plural rule for "{}" is not defined in '
'compare-locales'.format(lang))
self.plural_categories = ('one', 'other')
self.enforce_translated = enforce_translated
# Parsed input resources stored by resource path.
self.reference_resources = {}
self.localization_resources = {}
self.target_resources = {}
# An iterable of `FTL.Message` objects some of whose nodes can be the
# transform operations.
self.transforms = {}
# The evaluator instance is an AST transformer capable of walking an
# AST hierarchy and evaluating nodes which are migration Transforms.
self.evaluator = Evaluator(self)
def read_ftl_resource(self, path):
"""Read an FTL resource and parse it into an AST."""
f = codecs.open(path, 'r', 'utf8')
try:
contents = f.read()
except UnicodeDecodeError as err:
logger = logging.getLogger('migrate')
logger.warning('Unable to read file {}: {}'.format(path, err))
raise err
finally:
f.close()
ast = self.fluent_parser.parse(contents)
annots = [
annot
for entry in ast.body
if isinstance(entry, FTL.Junk)
for annot in entry.annotations
]
if len(annots):
logger = logging.getLogger('migrate')
for annot in annots:
msg = annot.message
logger.warning('Syntax error in {}: {}'.format(path, msg))
return ast
def read_legacy_resource(self, path):
"""Read a legacy resource and parse it into a dict."""
parser = getParser(path)
parser.readFile(path)
# Transform the parsed result which is an iterator into a dict.
return {
entity.key: entity.val for entity in parser
if entity.localized or self.enforce_translated
}
def read_reference_ftl(self, path):
"""Read and parse a reference FTL file.
A missing resource file is a fatal error and will raise an
UnreadableReferenceError.
"""
fullpath = os.path.join(self.reference_dir, path)
try:
return self.read_ftl_resource(fullpath)
except IOError:
error_message = 'Missing reference file: {}'.format(fullpath)
logging.getLogger('migrate').error(error_message)
raise UnreadableReferenceError(error_message)
except UnicodeDecodeError as err:
error_message = 'Error reading file {}: {}'.format(fullpath, err)
logging.getLogger('migrate').error(error_message)
raise UnreadableReferenceError(error_message)
def read_localization_ftl(self, path):
"""Read and parse an existing localization FTL file.
Create a new FTL.Resource if the file doesn't exist or can't be
decoded.
"""
fullpath = os.path.join(self.localization_dir, path)
try:
return self.read_ftl_resource(fullpath)
except IOError:
logger = logging.getLogger('migrate')
logger.info(
'Localization file {} does not exist and '
'it will be created'.format(path))
return FTL.Resource()
except UnicodeDecodeError:
logger = logging.getLogger('migrate')
logger.warning(
'Localization file {} has broken encoding. '
'It will be re-created and some translations '
'may be lost'.format(path))
return FTL.Resource()
def maybe_add_localization(self, path):
"""Add a localization resource to migrate translations from.
Uses a compare-locales parser to create a dict of (key, string value)
tuples.
For Fluent sources, we store the AST.
"""
try:
fullpath = os.path.join(self.localization_dir, path)
if not fullpath.endswith('.ftl'):
collection = self.read_legacy_resource(fullpath)
else:
collection = self.read_ftl_resource(fullpath)
except IOError:
logger = logging.getLogger('migrate')
logger.warning('Missing localization file: {}'.format(path))
else:
self.localization_resources[path] = collection
def get_legacy_source(self, path, key):
"""Get an entity value from a localized legacy source.
Used by the `Source` transform.
"""
resource = self.localization_resources[path]
return resource.get(key, None)
def get_fluent_source_pattern(self, path, key):
"""Get a pattern from a localized Fluent source.
If the key contains a `.`, does an attribute lookup.
Used by the `COPY_PATTERN` transform.
"""
resource = self.localization_resources[path]
msg_key, _, attr_key = key.partition('.')
found = None
for entry in resource.body:
if isinstance(entry, (FTL.Message, FTL.Term)):
if entry.id.name == msg_key:
found = entry
break
if found is None:
return None
if not attr_key:
return found.value
for attribute in found.attributes:
if attribute.id.name == attr_key:
return attribute.value
return None
def messages_equal(self, res1, res2):
"""Compare messages and terms of two FTL resources.
Uses FTL.BaseNode.equals to compare all messages/terms
in two FTL resources.
If the order or number of messages differ, the result is also False.
"""
def message_id(message):
"Return the message's identifer name for sorting purposes."
return message.id.name
messages1 = sorted(
(entry for entry in res1.body
if isinstance(entry, FTL.Message)
or isinstance(entry, FTL.Term)),
key=message_id)
messages2 = sorted(
(entry for entry in res2.body
if isinstance(entry, FTL.Message)
or isinstance(entry, FTL.Term)),
key=message_id)
for msg1, msg2 in zip_longest(messages1, messages2):
if msg1 is None or msg2 is None:
return False
if not msg1.equals(msg2):
return False
return True
def merge_changeset(self, changeset=None, known_translations=None):
"""Return a generator of FTL ASTs for the changeset.
The input data must be configured earlier using the `add_*` methods.
if given, `changeset` must be a set of (path, key) tuples describing
which legacy translations are to be merged. If `changeset` is None,
all legacy translations will be allowed to be migrated in a single
changeset.
We use the `in_changeset` method to determine if a message should be
migrated for the given changeset.
Given `changeset`, return a dict whose keys are resource paths and
values are `FTL.Resource` instances. The values will also be used to
update this context's existing localization resources.
"""
if changeset is None:
# Merge all known legacy translations. Used in tests.
changeset = {
(path, key)
for path, strings in self.localization_resources.items()
if not path.endswith('.ftl')
for key in strings.keys()
}
if known_translations is None:
known_translations = changeset
for path, reference in self.reference_resources.items():
current = self.target_resources[path]
transforms = self.transforms.get(path, [])
in_changeset = partial(
self.in_changeset, changeset, known_translations, path)
# Merge legacy translations with the existing ones using the
# reference as a template.
snapshot = merge_resource(
self, reference, current, transforms, in_changeset
)
# Skip this path if the messages in the merged snapshot are
# identical to those in the current state of the localization file.
# This may happen when:
#
# - none of the transforms is in the changset, or
# - all messages which would be migrated by the context's
# transforms already exist in the current state.
if self.messages_equal(current, snapshot):
continue
# Store the merged snapshot on the context so that the next merge
# already takes it into account as the existing localization.
self.target_resources[path] = snapshot
# The result for this path is a complete `FTL.Resource`.
yield path, snapshot
def in_changeset(self, changeset, known_translations, path, ident):
"""Check if a message should be migrated in this changeset.
The message is identified by path and ident.
A message will be migrated only if all of its dependencies
are present in the currently processed changeset.
If a transform defined for this message points to a missing
legacy translation, this message will not be merged. The
missing legacy dependency won't be present in the changeset.
This also means that partially translated messages (e.g.
constructed from two legacy strings out of which only one is
avaiable) will never be migrated.
"""
message_deps = self.dependencies.get((path, ident), None)
# Don't merge if we don't have a transform for this message.
if message_deps is None:
return False
# As a special case, if a transform exists but has no
# dependecies, it's a hardcoded `FTL.Node` which doesn't
# migrate any existing translation but rather creates a new
# one. Merge it.
if len(message_deps) == 0:
return True
# Make sure all the dependencies are present in the current
# changeset. Partial migrations are not currently supported.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=1321271
# We only return True if our current changeset touches
# the transform, and we have all of the dependencies.
active_deps = message_deps & changeset
available_deps = message_deps & known_translations
return active_deps and message_deps == available_deps
def serialize_changeset(self, changeset, known_translations=None):
"""Return a dict of serialized FTLs for the changeset.
Given `changeset`, return a dict whose keys are resource paths and
values are serialized FTL snapshots.
"""
return {
path: self.fluent_serializer.serialize(snapshot)
for path, snapshot in self.merge_changeset(
changeset, known_translations
)
}
def evaluate(self, node):
return self.evaluator.visit(node)
class MergeContext(object):
"""Stateful context for merging translation resources.
`MergeContext` must be configured with the target language and the
directory locations of the input data.
The transformation takes four types of input data:
- The en-US FTL reference files which will be used as templates for
message order, comments and sections.
- The current FTL files for the given language.
- The legacy (DTD, properties) translation files for the given
language. The translations from these files will be transformed
into FTL and merged into the existing FTL files for this language.
- A list of `FTL.Message` objects some of whose nodes are special
helper or transform nodes:
helpers: EXTERNAL_ARGUMENT, MESSAGE_REFERENCE
transforms: COPY, REPLACE_IN_TEXT, REPLACE, PLURALS, CONCAT
"""
def __init__(self, lang, reference_dir, localization_dir):
self.fluent_parser = FluentParser(with_spans=False)
self.fluent_serializer = FluentSerializer()
# An iterable of plural category names relevant to the context's
# language. E.g. ('one', 'other') for English.
try:
self.plural_categories = get_plural_categories(lang)
except RuntimeError as e:
print(e.message)
self.plural_categories = 'en'
# Paths to directories with input data, relative to CWD.
self.reference_dir = reference_dir
self.localization_dir = localization_dir
# Parsed input resources stored by resource path.
self.reference_resources = {}
self.localization_resources = {}
# An iterable of `FTL.Message` objects some of whose nodes can be the
# transform operations.
self.transforms = {}
# A dict whose keys are `(path, key)` tuples corresponding to target
# FTL translations, and values are sets of `(path, key)` tuples
# corresponding to localized entities which will be migrated.
self.dependencies = {}
def read_ftl_resource(self, path):
"""Read an FTL resource and parse it into an AST."""
f = codecs.open(path, 'r', 'utf8')
try:
contents = f.read()
except UnicodeDecodeError as err:
logger = logging.getLogger('migrate')
logger.warn('Unable to read file {}: {}'.format(path, err))
raise err
finally:
f.close()
ast = self.fluent_parser.parse(contents)
annots = [
annot
for entry in ast.body
for annot in entry.annotations
]
if len(annots):
logger = logging.getLogger('migrate')
for annot in annots:
msg = annot.message
logger.warn('Syntax error in {}: {}'.format(path, msg))
return ast
def read_legacy_resource(self, path):
"""Read a legacy resource and parse it into a dict."""
parser = getParser(path)
parser.readFile(path)
# Transform the parsed result which is an iterator into a dict.
return {entity.key: entity.val for entity in parser}
def maybe_add_localization(self, path):
"""Add a localization resource to migrate translations from.
Only legacy resources can be added as migration sources. The resource
may be missing on disk.
Uses a compare-locales parser to create a dict of (key, string value)
tuples.
"""
if path.endswith('.ftl'):
error_message = (
'Migrating translations from Fluent files is not supported '
'({})'.format(path))
logging.getLogger('migrate').error(error_message)
raise NotSupportedError(error_message)
try:
fullpath = os.path.join(self.localization_dir, path)
collection = self.read_legacy_resource(fullpath)
except IOError:
logger = logging.getLogger('migrate')
logger.warn('Missing localization file: {}'.format(path))
else:
self.localization_resources[path] = collection
def add_transforms(self, target, reference, transforms):
"""Define transforms for target using reference as template.
`target` is a path of the destination FTL file relative to the
localization directory. `reference` is a path to the template FTL
file relative to the reference directory.
Each transform is an extended FTL node with `Transform` nodes as some
values. Transforms are stored in their lazy AST form until
`merge_changeset` is called, at which point they are evaluated to real
FTL nodes with migrated translations.
Each transform is scanned for `Source` nodes which will be used to
build the list of dependencies for the transformed message.
"""
def get_sources(acc, cur):
if isinstance(cur, Source):
acc.add((cur.path, cur.key))
return acc
refpath = os.path.join(self.reference_dir, reference)
try:
ast = self.read_ftl_resource(refpath)
except IOError as err:
error_message = 'Missing reference file: {}'.format(refpath)
logging.getLogger('migrate').error(error_message)
raise UnreadableReferenceError(error_message)
except UnicodeDecodeError as err:
error_message = 'Error reading file {}: {}'.format(refpath, err)
logging.getLogger('migrate').error(error_message)
raise UnreadableReferenceError(error_message)
else:
# The reference file will be used by the merge function as
# a template for serializing the merge results.
self.reference_resources[target] = ast
for node in transforms:
# Scan `node` for `Source` nodes and collect the information they
# store into a set of dependencies.
dependencies = fold(get_sources, node, set())
# Set these sources as dependencies for the current transform.
self.dependencies[(target, node.id.name)] = dependencies
# Read all legacy translation files defined in Source transforms.
for path in set(path for path, _ in dependencies):
self.maybe_add_localization(path)
path_transforms = self.transforms.setdefault(target, [])
path_transforms += transforms
if target not in self.localization_resources:
fullpath = os.path.join(self.localization_dir, target)
try:
ast = self.read_ftl_resource(fullpath)
except IOError:
logger = logging.getLogger('migrate')
logger.info(
'Localization file {} does not exist and '
'it will be created'.format(target))
except UnicodeDecodeError:
logger = logging.getLogger('migrate')
logger.warn(
'Localization file {} will be re-created and some '
'translations might be lost'.format(target))
else:
self.localization_resources[target] = ast
def get_source(self, path, key):
"""Get an entity value from a localized legacy source.
Used by the `Source` transform.
"""
resource = self.localization_resources[path]
return resource.get(key, None)
def messages_equal(self, res1, res2):
"""Compare messages of two FTL resources.
Uses FTL.BaseNode.equals to compare all messages in two FTL resources.
If the order or number of messages differ, the result is also False.
"""
def message_id(message):
"Return the message's identifer name for sorting purposes."
return message.id.name
messages1 = sorted(
(entry for entry in res1.body if isinstance(entry, FTL.Message)),
key=message_id)
messages2 = sorted(
(entry for entry in res2.body if isinstance(entry, FTL.Message)),
key=message_id)
for msg1, msg2 in zip_longest(messages1, messages2):
if msg1 is None or msg2 is None:
return False
if not msg1.equals(msg2):
return False
return True
def merge_changeset(self, changeset=None):
"""Return a generator of FTL ASTs for the changeset.
The input data must be configured earlier using the `add_*` methods.
if given, `changeset` must be a set of (path, key) tuples describing
which legacy translations are to be merged.
Given `changeset`, return a dict whose keys are resource paths and
values are `FTL.Resource` instances. The values will also be used to
update this context's existing localization resources.
"""
if changeset is None:
# Merge all known legacy translations. Used in tests.
changeset = {
(path, key)
for path, strings in self.localization_resources.iteritems()
if not path.endswith('.ftl')
for key in strings.iterkeys()
}
for path, reference in self.reference_resources.iteritems():
current = self.localization_resources.get(path, FTL.Resource())
transforms = self.transforms.get(path, [])
def in_changeset(ident):
"""Check if entity should be merged.
If at least one dependency of the entity is in the current
set of changeset, merge it.
"""
message_deps = self.dependencies.get((path, ident), None)
# Don't merge if we don't have a transform for this message.
if message_deps is None:
return False
# As a special case, if a transform exists but has no
# dependecies, it's a hardcoded `FTL.Node` which doesn't
# migrate any existing translation but rather creates a new
# one. Merge it.
if len(message_deps) == 0:
return True
# If the intersection of the dependencies and the current
# changeset is non-empty, merge this message.
return message_deps & changeset
# Merge legacy translations with the existing ones using the
# reference as a template.
snapshot = merge_resource(
self, reference, current, transforms, in_changeset
)
# Skip this path if the messages in the merged snapshot are
# identical to those in the current state of the localization file.
# This may happen when:
#
# - none of the transforms is in the changset, or
# - all messages which would be migrated by the context's
# transforms already exist in the current state.
if self.messages_equal(current, snapshot):
continue
# Store the merged snapshot on the context so that the next merge
# already takes it into account as the existing localization.
self.localization_resources[path] = snapshot
# The result for this path is a complete `FTL.Resource`.
yield path, snapshot
def serialize_changeset(self, changeset):
"""Return a dict of serialized FTLs for the changeset.
Given `changeset`, return a dict whose keys are resource paths and
values are serialized FTL snapshots.
"""
return {
path: self.fluent_serializer.serialize(snapshot)
for path, snapshot in self.merge_changeset(changeset)
}
def parse_literal(input):
parser = FluentParser(with_spans=False)
ast = parser.parse_entry(input)
expr = ast.value.elements[0].expression
return expr.parse()
class MergeContext(object):
"""Stateful context for merging translation resources.
`MergeContext` must be configured with the target language and the
directory locations of the input data.
The transformation takes four types of input data:
- The en-US FTL reference files which will be used as templates for
message order, comments and sections.
- The current FTL files for the given language.
- The legacy (DTD, properties) translation files for the given
language. The translations from these files will be transformed
into FTL and merged into the existing FTL files for this language.
- A list of `FTL.Message` objects some of whose nodes are special
helper or transform nodes:
helpers: LITERAL, EXTERNAL_ARGUMENT, MESSAGE_REFERENCE
transforms: COPY, REPLACE_IN_TEXT, REPLACE, PLURALS, CONCAT
"""
def __init__(self, lang, reference_dir, localization_dir):
self.fluent_parser = FluentParser(with_spans=False)
self.fluent_serializer = FluentSerializer()
# An iterable of plural category names relevant to the context's
# language. E.g. ('one', 'other') for English.
try:
self.plural_categories = get_plural_categories(lang)
except RuntimeError as e:
print(e.message)
self.plural_categories = 'en'
# Paths to directories with input data, relative to CWD.
self.reference_dir = reference_dir
self.localization_dir = localization_dir
# Parsed input resources stored by resource path.
self.reference_resources = {}
self.localization_resources = {}
# An iterable of `FTL.Message` objects some of whose nodes can be the
# transform operations.
self.transforms = {}
# A dict whose keys are `(path, key)` tuples corresponding to target
# FTL translations, and values are sets of `(path, key)` tuples
# corresponding to localized entities which will be migrated.
self.dependencies = {}
def read_ftl_resource(self, path):
"""Read an FTL resource and parse it into an AST."""
f = codecs.open(path, 'r', 'utf8')
try:
contents = f.read()
finally:
f.close()
ast = self.fluent_parser.parse(contents)
annots = [annot for entry in ast.body for annot in entry.annotations]
if len(annots):
logger = logging.getLogger('migrate')
for annot in annots:
msg = annot.message
logger.warn(u'Syntax error in {}: {}'.format(path, msg))
return ast
def read_legacy_resource(self, path):
"""Read a legacy resource and parse it into a dict."""
parser = getParser(path)
parser.readFile(path)
# Transform the parsed result which is an iterator into a dict.
return {entity.key: entity.val for entity in parser}
def add_reference(self, path, realpath=None):
"""Add an FTL AST to this context's reference resources."""
fullpath = os.path.join(self.reference_dir, realpath or path)
try:
ast = self.read_ftl_resource(fullpath)
except IOError as err:
logger = logging.getLogger('migrate')
logger.error(u'Missing reference file: {}'.format(path))
raise err
except UnicodeDecodeError as err:
logger = logging.getLogger('migrate')
logger.error(u'Error reading file {}: {}'.format(path, err))
raise err
else:
self.reference_resources[path] = ast
def add_localization(self, path):
"""Add an existing localization resource.
If it's an FTL resource, add an FTL AST. Otherwise, it's a legacy
resource. Use a compare-locales parser to create a dict of (key,
string value) tuples.
"""
fullpath = os.path.join(self.localization_dir, path)
if fullpath.endswith('.ftl'):
try:
ast = self.read_ftl_resource(fullpath)
except IOError:
logger = logging.getLogger('migrate')
logger.warn(u'Missing localization file: {}'.format(path))
except UnicodeDecodeError as err:
logger = logging.getLogger('migrate')
logger.warn(u'Error reading file {}: {}'.format(path, err))
else:
self.localization_resources[path] = ast
else:
try:
collection = self.read_legacy_resource(fullpath)
except IOError:
logger = logging.getLogger('migrate')
logger.warn(u'Missing localization file: {}'.format(path))
else:
self.localization_resources[path] = collection
def add_transforms(self, path, transforms):
"""Define transforms for path.
Each transform is an extended FTL node with `Transform` nodes as some
values. Transforms are stored in their lazy AST form until
`merge_changeset` is called, at which point they are evaluated to real
FTL nodes with migrated translations.
Each transform is scanned for `Source` nodes which will be used to
build the list of dependencies for the transformed message.
"""
def get_sources(acc, cur):
if isinstance(cur, Source):
acc.add((cur.path, cur.key))
return acc
for node in transforms:
# Scan `node` for `Source` nodes and collect the information they
# store into a set of dependencies.
dependencies = fold(get_sources, node, set())
# Set these sources as dependencies for the current transform.
self.dependencies[(path, node.id.name)] = dependencies
path_transforms = self.transforms.setdefault(path, [])
path_transforms += transforms
def get_source(self, path, key):
"""Get an entity value from the localized source.
Used by the `Source` transform.
"""
if path.endswith('.ftl'):
resource = self.localization_resources[path]
return get_message(resource.body, key)
else:
resource = self.localization_resources[path]
return resource.get(key, None)
def merge_changeset(self, changeset=None):
"""Return a generator of FTL ASTs for the changeset.
The input data must be configured earlier using the `add_*` methods.
if given, `changeset` must be a set of (path, key) tuples describing
which legacy translations are to be merged.
Given `changeset`, return a dict whose keys are resource paths and
values are `FTL.Resource` instances. The values will also be used to
update this context's existing localization resources.
"""
if changeset is None:
# Merge all known legacy translations.
changeset = {
(path, key)
for path, strings in self.localization_resources.iteritems()
for key in strings.iterkeys()
}
for path, reference in self.reference_resources.iteritems():
current = self.localization_resources.get(path, FTL.Resource())
transforms = self.transforms.get(path, [])
def in_changeset(ident):
"""Check if entity should be merged.
If at least one dependency of the entity is in the current
set of changeset, merge it.
"""
message_deps = self.dependencies.get((path, ident), None)
# Don't merge if we don't have a transform for this message.
if message_deps is None:
return False
# As a special case, if a transform exists but has no
# dependecies, it's a hardcoded `FTL.Node` which doesn't
# migrate any existing translation but rather creates a new
# one. Merge it.
if len(message_deps) == 0:
return True
# If the intersection of the dependencies and the current
# changeset is non-empty, merge this message.
return message_deps & changeset
# Merge legacy translations with the existing ones using the
# reference as a template.
snapshot = merge_resource(self, reference, current, transforms,
in_changeset)
# If none of the transforms is in the given changeset, the merged
# snapshot is identical to the current translation. We compare
# JSON trees rather then use filtering by `in_changeset` to account
# for translations removed from `reference`.
if snapshot.to_json() == current.to_json():
continue
# Store the merged snapshot on the context so that the next merge
# already takes it into account as the existing localization.
self.localization_resources[path] = snapshot
# The result for this path is a complete `FTL.Resource`.
yield path, snapshot
def serialize_changeset(self, changeset):
"""Return a dict of serialized FTLs for the changeset.
Given `changeset`, return a dict whose keys are resource paths and
values are serialized FTL snapshots.
"""
return {
path: self.fluent_serializer.serialize(snapshot)
for path, snapshot in self.merge_changeset(changeset)
}
def setUp(self):
self.parser = FluentParser(with_spans=False)
class TestParseEntry(unittest.TestCase):
maxDiff = None
def setUp(self):
self.parser = FluentParser(with_spans=False)
def test_simple_message(self):
input = """\
foo = Foo
"""
output = {
"comment": None,
"value": {
"elements": [{
"span": None,
"type": "TextElement",
"value": "Foo"
}],
"span": None,
"type": "Pattern"
},
"attributes": [],
"type": "Message",
"span": None,
"id": {
"span": None,
"type": "Identifier",
"name": "foo"
}
}
message = self.parser.parse_entry(dedent_ftl(input))
self.assertEqual(message.to_json(), output)
def test_ignore_attached_comment(self):
input = """\
# Attached Comment
foo = Foo
"""
output = {
"comment": None,
"value": {
"elements": [{
"span": None,
"type": "TextElement",
"value": "Foo"
}],
"span": None,
"type": "Pattern"
},
"attributes": [],
"type": "Message",
"id": {
"name": "foo",
"span": None,
"type": "Identifier"
},
"span": None,
"type": "Message"
}
message = self.parser.parse_entry(dedent_ftl(input))
self.assertEqual(message.to_json(), output)
def test_return_junk(self):
input = """\
# Attached Comment
junk
"""
output = {
"content":
"junk\n",
"annotations": [{
"arguments": ["="],
"code": "E0003",
"message": "Expected token: \"=\"",
"span": {
"end": 23,
"start": 23,
"type": "Span"
},
"type": "Annotation"
}],
"span":
None,
"type":
"Junk"
}
message = self.parser.parse_entry(dedent_ftl(input))
self.assertEqual(message.to_json(), output)
def test_ignore_all_valid_comments(self):
input = """\
# Attached Comment
## Group Comment
### Resource Comment
foo = Foo
"""
output = {
"comment": None,
"value": {
"elements": [{
"span": None,
"type": "TextElement",
"value": "Foo"
}],
"span": None,
"type": "Pattern"
},
"attributes": [],
"span": None,
"type": "Message",
"id": {
"name": "foo",
"span": None,
"type": "Identifier"
}
}
message = self.parser.parse_entry(dedent_ftl(input))
self.assertEqual(message.to_json(), output)
def test_do_not_ignore_invalid_comments(self):
input = """\
# Attached Comment
##Invalid Comment
"""
output = {
"content":
"##Invalid Comment\n",
"annotations": [{
"arguments": [" "],
"code": "E0003",
"message": "Expected token: \" \"",
"span": {
"end": 21,
"start": 21,
"type": "Span"
},
"type": "Annotation"
}],
"span":
None,
"type":
"Junk"
}
message = self.parser.parse_entry(dedent_ftl(input))
self.assertEqual(message.to_json(), output)
def setUp(self):
self.parser = FluentParser()
ru_file_full_path = en_analog_file_path.replace('en-US', 'ru-RU')
if os.path.isfile(ru_file_full_path):
return
else:
en_file = FluentFile(f'{en_analog_file_path}')
file = FluentFile(f'{ru_file_full_path}')
file.save_data(en_file.read_data())
logging.info(f'Создан файл русской локали {ru_file_full_path}')
return ru_file_full_path
######################################## Var definitions ###############################################################
logging.basicConfig(level=logging.INFO)
project = Project()
serializer = FluentSerializer()
parser = FluentParser()
formatter = FluentFormatter()
yaml_files_paths = project.get_files_paths_by_dir(project.prototypes_dir_path,
'yml')
yaml_files = list(
map(lambda yaml_file_path: YAMLFile(yaml_file_path), yaml_files_paths))
########################################################################################################################
logging.info(f'Поиск yaml-файлов ...')
YAMLExtractor(yaml_files).execute()