def get_identifiers(fileData): idents = [] for message in parse(fileData).body: if not isinstance(message, ast.Message): continue idents.append(message.id.name) sorted(idents) return idents
def add_simple_message(fname, key, message): orig = "" if os.path.exists(fname): orig = open(fname).read() obj = parse(orig) obj.body.append(Message(Identifier(key), Pattern([TextElement(message)]))) modified = serialize(obj) open(fname, "w").write(modified)
def test(self): ftl_path = os.path.join(fixtures, file_name + '.ftl') ast_path = os.path.join(fixtures, file_name + '.json') source = read_file(ftl_path) expected = read_file(ast_path) ast = parse(source) self.assertEqual(ast.to_json(remove_untested), json.loads(expected))
def lint(paths, config, fix=None, **lintargs): files = list(expand_exclusions(paths, config, lintargs["root"])) exclusions = get_exclusions(lintargs["root"]) results = [] for path in files: contents = open(path, "r", encoding="utf-8").read() linter = Linter(path, config, exclusions, contents, get_offsets_and_lines(contents)) linter.visit(parse(contents)) results.extend(linter.results) return results
def get_entry(fname, key): if not os.path.exists(fname): return with open(fname) as file: orig = file.read() obj = parse(orig) for ent in obj.body: if isinstance(ent, Junk): raise Exception(f"file had junk! {fname} {ent}") elif isinstance(ent, Message): if ent.id.name == old_key: return copy.deepcopy(ent)
def add_simple_message(fname, key, message): orig = "" if os.path.exists(fname): orig = open(fname).read() obj = parse(orig) for ent in obj.body: if isinstance(ent, Junk): raise Exception(f"file had junk! {fname} {ent}") obj.body.append(Message(Identifier(key), Pattern([TextElement(message)]))) modified = serialize(obj, with_junk=True) # escape leading dots modified = re.sub(r"(?ms)^( +)\.", "\\1{\".\"}", modified) # ensure the resulting serialized file is valid by parsing again obj = parse(modified) for ent in obj.body: if isinstance(ent, Junk): raise Exception(f"introduced junk! {fname} {ent}") # it's ok, write it out open(fname, "w").write(modified)
def dupe_key(fname, old, new): if not os.path.exists(fname): return with open(fname) as file: orig = file.read() obj = parse(orig) for ent in obj.body: if isinstance(ent, Junk): raise Exception(f"file had junk! {fname} {ent}") # locate the existing key for item in obj.body: if isinstance(item, Message): if item.id.name == old: item2 = copy.deepcopy(item) item2.id.name = new obj.body.append(item2) break # print(item.id.name) # print(item.value.elements) # for e in item.value.elements: # print(e.value) modified = serialize(obj, with_junk=True) # escape leading dots modified = re.sub(r"(?ms)^( +)\.", '\\1{"."}', modified) # ensure the resulting serialized file is valid by parsing again obj = parse(modified) for ent in obj.body: if isinstance(ent, Junk): raise Exception(f"introduced junk! {fname} {ent}") # it's ok, write it out with open(fname, "w") as file: file.write(modified)
def check_file(path: str, fix: bool) -> bool: "True if file is ok." orig_text = open(path, encoding="utf8").read() obj = parse(orig_text, with_spans=False) # make sure there's no junk for ent in obj.body: if isinstance(ent, Junk): raise Exception(f"file had junk! {path} {ent}") # serialize new_text = serialize(obj) # make sure serializing did not introduce new junk obj = parse(new_text, with_spans=False) for ent in obj.body: if isinstance(ent, Junk): raise Exception(f"file introduced junk! {path} {ent}") if new_text == orig_text: return check_missing_terms(path) if fix: print(f"Fixing {path}") open(path, "w", newline="\n", encoding="utf8").write(new_text) return True else: print(f"Bad formatting in {path}") print( "\n".join( difflib.unified_diff( orig_text.splitlines(), new_text.splitlines(), fromfile="bad", tofile="good", lineterm="", ) ) ) return False
def test(self): ftl_path = os.path.join(fixtures, file_name + '.ftl') ftl_file = read_file(ftl_path) [expected_directives, source] = preprocess(ftl_file) expected = '{}\n'.format('\n'.join(expected_directives)) ast = parse(source) actual_directives = [ serialize_annotation(annot) for entry in ast.body if isinstance(entry, ftl.Junk) for annot in entry.annotations ] actual = '{}\n'.format('\n'.join(actual_directives)) self.assertEqual(actual, expected)
def transform_string_in_file(path): obj = parse(open(path).read(), with_spans=False) changed = False for ent in obj.body: if isinstance(ent, Junk): raise Exception(f"file had junk! {path} {ent}") if getattr(ent, "id", None): key = ent.id.name for (target_key, src, dst) in target_repls: if key == target_key: for elem in ent.value.elements: newval = elem.value.replace(src, dst) if newval != elem.value: elem.value = newval changed = True if changed: open(path, "w", encoding="utf8").write(serialize(obj)) print("updated", path)
def find_error_ids(filename, known_strings): with open(filename, "r", encoding="utf-8") as f: known_strings += [ m.id.name for m in parse(f.read()).body if isinstance(m, Message) ]
def print_ast(fileType, data): ast = parse(data) print(json.dumps(ast.to_json(), indent=2, ensure_ascii=False))
def file(self, parent, ref, l10n, locale): if not os.path.isfile(ref): return with open(ref) as fh: ref_entries = parse(fh.read(), with_spans=False).body l10n_entries = {} if os.path.isfile(l10n): with open(l10n) as fh: l10n_entries = { ("-" if isinstance(e, ast.Term) else "") + e.id.name: e for e in parse(fh.read(), with_spans=False).body if isinstance(e, (ast.Message, ast.Term)) } root = parent indent = 1 group_id = 1 for entry in ref_entries: if isinstance(entry, ast.GroupComment): parent = root if not entry.content: continue parent = ET.SubElement(parent, f"{{{xmlns}}}group", {"id": f"group-{group_id}"}) group_id += 1 parent.text = "\n " parent.tail = "\n " notes = ET.SubElement(parent, f"{{{xmlns}}}notes") notes.tail = "\n " note = ET.SubElement(notes, f"{{{xmlns}}}note") note.text = entry.content elif isinstance(entry, (ast.Message, ast.Term)): entry_id = "-" if isinstance(entry, ast.Term) else "" entry_id += entry.id.name unit = ET.SubElement(parent, f"{{{xmlns}}}unit", {"id": entry_id}) unit.text = "\n" + (indent + 1) * " " unit.tail = "\n" + indent * " " if entry.comment: notes = ET.SubElement(unit, f"{{{xmlns}}}notes") notes.tail = "\n" + (indent + 1) * " " note = ET.SubElement(notes, f"{{{xmlns}}}note") note.text = entry.comment.content segment = ET.SubElement(unit, f"{{{xmlns}}}segment") segment.tail = segment.text = "\n" + (indent + 1) * " " source = ET.SubElement(segment, f"{{{xmlns}}}source") source.tail = "\n" + (indent + 1) * " " content = serializer.serialize_pattern(entry.value) # serialize_pattern prefixes inline patterns with " ", which we don't want if content[0] == " ": content = content[1:] source.text = content if entry_id not in l10n_entries: continue target = ET.SubElement(segment, f"{{{xmlns}}}target") target.tail = "\n" + (indent + 1) * " " content = serializer.serialize_pattern( l10n_entries[entry_id].value) # serialize_pattern prefixes inline patterns with " ", which we don't want if content[0] == " ": content = content[1:] target.text = content
""" Tool to extract core strings and keys from .ftl files. """ import glob import json import os from fluent.syntax import parse from fluent.syntax.ast import Junk from fluent.syntax.serializer import serialize_element root = os.environ["BUILD_WORKSPACE_DIRECTORY"] ftl_files = glob.glob(os.path.join(root, "ftl", "core", "*.ftl"), recursive=True) keys_by_value = {} for path in ftl_files: obj = parse(open(path).read(), with_spans=False) for ent in obj.body: if isinstance(ent, Junk): raise Exception(f"file had junk! {path} {ent}") if getattr(ent, "id", None): key = ent.id.name val = "".join(serialize_element(elem) for elem in ent.value.elements) if val in keys_by_value: print("duplicate found:", keys_by_value[val], key) keys_by_value.setdefault(val, []).append(key) json.dump(keys_by_value, open(os.path.join(root, "keys_by_value.json"), "w")) print("keys:", len(keys_by_value))
def pretty_print(fileType, data): ast = parse(data) print(serialize(ast))
def __init__(self, text): self.text = text self.ast = parse(text)
def parse(self, fluentsrc): resource = parse(fluentsrc.decode("utf-8")) for entry in resource.body: self.addunit(FluentUnit(entry=entry))