def get_identifiers(fileData):
idents = []
for message in parse(fileData).body:
if not isinstance(message, ast.Message):
continue
idents.append(message.id.name)
sorted(idents)
return idents
def add_simple_message(fname, key, message):
orig = ""
if os.path.exists(fname):
orig = open(fname).read()
obj = parse(orig)
obj.body.append(Message(Identifier(key), Pattern([TextElement(message)])))
modified = serialize(obj)
open(fname, "w").write(modified)
def test(self):
ftl_path = os.path.join(fixtures, file_name + '.ftl')
ast_path = os.path.join(fixtures, file_name + '.json')
source = read_file(ftl_path)
expected = read_file(ast_path)
ast = parse(source)
self.assertEqual(ast.to_json(remove_untested),
json.loads(expected))
def lint(paths, config, fix=None, **lintargs):
files = list(expand_exclusions(paths, config, lintargs["root"]))
exclusions = get_exclusions(lintargs["root"])
results = []
for path in files:
contents = open(path, "r", encoding="utf-8").read()
linter = Linter(path, config, exclusions, contents,
get_offsets_and_lines(contents))
linter.visit(parse(contents))
results.extend(linter.results)
return results
def get_entry(fname, key):
if not os.path.exists(fname):
return
with open(fname) as file:
orig = file.read()
obj = parse(orig)
for ent in obj.body:
if isinstance(ent, Junk):
raise Exception(f"file had junk! {fname} {ent}")
elif isinstance(ent, Message):
if ent.id.name == old_key:
return copy.deepcopy(ent)
def add_simple_message(fname, key, message):
orig = ""
if os.path.exists(fname):
orig = open(fname).read()
obj = parse(orig)
for ent in obj.body:
if isinstance(ent, Junk):
raise Exception(f"file had junk! {fname} {ent}")
obj.body.append(Message(Identifier(key), Pattern([TextElement(message)])))
modified = serialize(obj, with_junk=True)
# escape leading dots
modified = re.sub(r"(?ms)^( +)\.", "\\1{\".\"}", modified)
# ensure the resulting serialized file is valid by parsing again
obj = parse(modified)
for ent in obj.body:
if isinstance(ent, Junk):
raise Exception(f"introduced junk! {fname} {ent}")
# it's ok, write it out
open(fname, "w").write(modified)
def dupe_key(fname, old, new):
if not os.path.exists(fname):
return
with open(fname) as file:
orig = file.read()
obj = parse(orig)
for ent in obj.body:
if isinstance(ent, Junk):
raise Exception(f"file had junk! {fname} {ent}")
# locate the existing key
for item in obj.body:
if isinstance(item, Message):
if item.id.name == old:
item2 = copy.deepcopy(item)
item2.id.name = new
obj.body.append(item2)
break
# print(item.id.name)
# print(item.value.elements)
# for e in item.value.elements:
# print(e.value)
modified = serialize(obj, with_junk=True)
# escape leading dots
modified = re.sub(r"(?ms)^( +)\.", '\\1{"."}', modified)
# ensure the resulting serialized file is valid by parsing again
obj = parse(modified)
for ent in obj.body:
if isinstance(ent, Junk):
raise Exception(f"introduced junk! {fname} {ent}")
# it's ok, write it out
with open(fname, "w") as file:
file.write(modified)
def check_file(path: str, fix: bool) -> bool:
"True if file is ok."
orig_text = open(path, encoding="utf8").read()
obj = parse(orig_text, with_spans=False)
# make sure there's no junk
for ent in obj.body:
if isinstance(ent, Junk):
raise Exception(f"file had junk! {path} {ent}")
# serialize
new_text = serialize(obj)
# make sure serializing did not introduce new junk
obj = parse(new_text, with_spans=False)
for ent in obj.body:
if isinstance(ent, Junk):
raise Exception(f"file introduced junk! {path} {ent}")
if new_text == orig_text:
return check_missing_terms(path)
if fix:
print(f"Fixing {path}")
open(path, "w", newline="\n", encoding="utf8").write(new_text)
return True
else:
print(f"Bad formatting in {path}")
print(
"\n".join(
difflib.unified_diff(
orig_text.splitlines(),
new_text.splitlines(),
fromfile="bad",
tofile="good",
lineterm="",
)
)
)
return False
def test(self):
ftl_path = os.path.join(fixtures, file_name + '.ftl')
ftl_file = read_file(ftl_path)
[expected_directives, source] = preprocess(ftl_file)
expected = '{}\n'.format('\n'.join(expected_directives))
ast = parse(source)
actual_directives = [
serialize_annotation(annot) for entry in ast.body
if isinstance(entry, ftl.Junk)
for annot in entry.annotations
]
actual = '{}\n'.format('\n'.join(actual_directives))
self.assertEqual(actual, expected)
def transform_string_in_file(path):
obj = parse(open(path).read(), with_spans=False)
changed = False
for ent in obj.body:
if isinstance(ent, Junk):
raise Exception(f"file had junk! {path} {ent}")
if getattr(ent, "id", None):
key = ent.id.name
for (target_key, src, dst) in target_repls:
if key == target_key:
for elem in ent.value.elements:
newval = elem.value.replace(src, dst)
if newval != elem.value:
elem.value = newval
changed = True
if changed:
open(path, "w", encoding="utf8").write(serialize(obj))
print("updated", path)
def find_error_ids(filename, known_strings):
with open(filename, "r", encoding="utf-8") as f:
known_strings += [
m.id.name for m in parse(f.read()).body if isinstance(m, Message)
]
def print_ast(fileType, data):
ast = parse(data)
print(json.dumps(ast.to_json(), indent=2, ensure_ascii=False))
def file(self, parent, ref, l10n, locale):
if not os.path.isfile(ref):
return
with open(ref) as fh:
ref_entries = parse(fh.read(), with_spans=False).body
l10n_entries = {}
if os.path.isfile(l10n):
with open(l10n) as fh:
l10n_entries = {
("-" if isinstance(e, ast.Term) else "") + e.id.name: e
for e in parse(fh.read(), with_spans=False).body
if isinstance(e, (ast.Message, ast.Term))
}
root = parent
indent = 1
group_id = 1
for entry in ref_entries:
if isinstance(entry, ast.GroupComment):
parent = root
if not entry.content:
continue
parent = ET.SubElement(parent, f"{{{xmlns}}}group",
{"id": f"group-{group_id}"})
group_id += 1
parent.text = "\n "
parent.tail = "\n "
notes = ET.SubElement(parent, f"{{{xmlns}}}notes")
notes.tail = "\n "
note = ET.SubElement(notes, f"{{{xmlns}}}note")
note.text = entry.content
elif isinstance(entry, (ast.Message, ast.Term)):
entry_id = "-" if isinstance(entry, ast.Term) else ""
entry_id += entry.id.name
unit = ET.SubElement(parent, f"{{{xmlns}}}unit",
{"id": entry_id})
unit.text = "\n" + (indent + 1) * " "
unit.tail = "\n" + indent * " "
if entry.comment:
notes = ET.SubElement(unit, f"{{{xmlns}}}notes")
notes.tail = "\n" + (indent + 1) * " "
note = ET.SubElement(notes, f"{{{xmlns}}}note")
note.text = entry.comment.content
segment = ET.SubElement(unit, f"{{{xmlns}}}segment")
segment.tail = segment.text = "\n" + (indent + 1) * " "
source = ET.SubElement(segment, f"{{{xmlns}}}source")
source.tail = "\n" + (indent + 1) * " "
content = serializer.serialize_pattern(entry.value)
# serialize_pattern prefixes inline patterns with " ", which we don't want
if content[0] == " ":
content = content[1:]
source.text = content
if entry_id not in l10n_entries:
continue
target = ET.SubElement(segment, f"{{{xmlns}}}target")
target.tail = "\n" + (indent + 1) * " "
content = serializer.serialize_pattern(
l10n_entries[entry_id].value)
# serialize_pattern prefixes inline patterns with " ", which we don't want
if content[0] == " ":
content = content[1:]
target.text = content
"""
Tool to extract core strings and keys from .ftl files.
"""
import glob
import json
import os
from fluent.syntax import parse
from fluent.syntax.ast import Junk
from fluent.syntax.serializer import serialize_element
root = os.environ["BUILD_WORKSPACE_DIRECTORY"]
ftl_files = glob.glob(os.path.join(root, "ftl", "core", "*.ftl"), recursive=True)
keys_by_value = {}
for path in ftl_files:
obj = parse(open(path).read(), with_spans=False)
for ent in obj.body:
if isinstance(ent, Junk):
raise Exception(f"file had junk! {path} {ent}")
if getattr(ent, "id", None):
key = ent.id.name
val = "".join(serialize_element(elem) for elem in ent.value.elements)
if val in keys_by_value:
print("duplicate found:", keys_by_value[val], key)
keys_by_value.setdefault(val, []).append(key)
json.dump(keys_by_value, open(os.path.join(root, "keys_by_value.json"), "w"))
print("keys:", len(keys_by_value))
def pretty_print(fileType, data):
ast = parse(data)
print(serialize(ast))
def __init__(self, text):
self.text = text
self.ast = parse(text)
def parse(self, fluentsrc):
resource = parse(fluentsrc.decode("utf-8"))
for entry in resource.body:
self.addunit(FluentUnit(entry=entry))
