def compareProjects(project_configs, stat_observer=None, file_stats=False, merge_stage=None, clobber_merge=False): locales = set() observers = [] for project in project_configs: observers.append(Observer(filter=project.filter, file_stats=file_stats)) locales.update(project.locales) if stat_observer is not None: stat_observers = [stat_observer] else: stat_observers = None comparer = ContentComparer(observers, stat_observers=stat_observers) for locale in sorted(locales): files = paths.ProjectFiles(locale, project_configs, mergebase=merge_stage) root = mozpath.commonprefix([m['l10n'].prefix for m in files.matchers]) if merge_stage is not None: if clobber_merge: mergematchers = set(_m.get('merge') for _m in files.matchers) mergematchers.discard(None) for matcher in mergematchers: clobberdir = matcher.prefix if os.path.exists(clobberdir): shutil.rmtree(clobberdir) print "clobbered " + clobberdir for l10npath, refpath, mergepath, extra_tests in files: # module and file path are needed for legacy filter.py support module = None fpath = mozpath.relpath(l10npath, root) for _m in files.matchers: if _m['l10n'].match(l10npath): if _m['module']: # legacy ini support, set module, and resolve # local path against the matcher prefix, # which includes the module module = _m['module'] fpath = mozpath.relpath(l10npath, _m['l10n'].prefix) break reffile = paths.File(refpath, fpath or refpath, module=module) l10n = paths.File(l10npath, fpath or l10npath, module=module, locale=locale) if not os.path.exists(l10npath): comparer.add(reffile, l10n) continue if not os.path.exists(refpath): comparer.remove(l10n) continue comparer.compare(reffile, l10n, mergepath, extra_tests) return observers
def compareProjects(project_configs, stat_observer=None, file_stats=False, merge_stage=None, clobber_merge=False): locales = set() observers = [] for project in project_configs: observers.append( Observer(filter=project.filter, file_stats=file_stats)) locales.update(project.locales) if stat_observer is not None: stat_observers = [stat_observer] else: stat_observers = None comparer = ContentComparer(observers, stat_observers=stat_observers) for locale in sorted(locales): files = paths.ProjectFiles(locale, project_configs, mergebase=merge_stage) root = mozpath.commonprefix([m['l10n'].prefix for m in files.matchers]) if merge_stage is not None: if clobber_merge: mergematchers = set(_m.get('merge') for _m in files.matchers) mergematchers.discard(None) for matcher in mergematchers: clobberdir = matcher.prefix if os.path.exists(clobberdir): shutil.rmtree(clobberdir) print "clobbered " + clobberdir for l10npath, refpath, mergepath, extra_tests in files: # module and file path are needed for legacy filter.py support module = None fpath = mozpath.relpath(l10npath, root) for _m in files.matchers: if _m['l10n'].match(l10npath): if _m['module']: # legacy ini support, set module, and resolve # local path against the matcher prefix, # which includes the module module = _m['module'] fpath = mozpath.relpath(l10npath, _m['l10n'].prefix) break reffile = paths.File(refpath, fpath or refpath, module=module) l10n = paths.File(l10npath, fpath or l10npath, module=module, locale=locale) if not os.path.exists(l10npath): comparer.add(reffile, l10n) continue if not os.path.exists(refpath): comparer.remove(l10n) continue comparer.compare(reffile, l10n, mergepath, extra_tests) return observers
def __init__(self, repo, rev, projects): self.repo = repo self.ctx = repo[rev] self.root = repo.root() self.manifest = None self.configs_map = {} # get paths for our TOML files for p in projects: all_configpaths = { mozpath.abspath(c.path).encode("utf-8") for c in p.configs } for refpath in all_configpaths: local_path = mozpath.relpath(refpath, self.root) if local_path not in self.ctx: print("ignoring", refpath) continue targetpath = b"/".join(( expand(None, "{l10n_base}", p.environ).encode("utf-8"), b"en-US", generate_filename(local_path), )) self.configs_map[refpath] = targetpath super(HGFiles, self).__init__("en-US", projects) for m in self.matchers: m["l10n"].encoding = "utf-8" if "reference" in m: m["reference"].encoding = "utf-8" if self.exclude: for m in self.exclude.matchers: m["l10n"].encoding = "utf-8" if "reference" in m: m["reference"].encoding = "utf-8"
def main(): p = argparse.ArgumentParser( description='Validate localizable strings', epilog=epilog, ) p.add_argument('l10n_toml') p.add_argument('--version', action='version', version='%(prog)s ' + version) p.add_argument('-W', action='store_true', help='error on warnings') p.add_argument( '--l10n-reference', dest='l10n_reference', metavar='PATH', help='check for conflicts against an l10n-only reference repository ' 'like gecko-strings', ) p.add_argument( '--reference-project', dest='ref_project', metavar='PATH', help='check for conflicts against a reference project like ' 'android-l10n', ) args = p.parse_args() if args.l10n_reference: l10n_base, locale = \ os.path.split(os.path.abspath(args.l10n_reference)) if not locale or not os.path.isdir(args.l10n_reference): p.error('Pass an existing l10n reference') else: l10n_base = '.' locale = None pc = paths.TOMLParser().parse(args.l10n_toml, env={'l10n_base': l10n_base}) if locale: pc.set_locales([locale], deep=True) files = paths.ProjectFiles(locale, [pc]) get_reference_and_tests = default_reference_and_tests if args.l10n_reference: get_reference_and_tests = l10n_base_reference_and_tests(files) elif args.ref_project: get_reference_and_tests = mirror_reference_and_tests( files, args.ref_project) linter = L10nLinter() results = linter.lint( (f for f, _, _, _ in files.iter_reference() if parser.hasParser(f)), get_reference_and_tests) rv = 0 if results: rv = 1 if all(r['level'] == 'warning' for r in results) and not args.W: rv = 0 for result in results: print('{} ({}:{}): {}'.format(mozpath.relpath(result['path'], '.'), result.get('lineno', 0), result.get('column', 0), result['message'])) return rv
def load(self, parse_ctx): try: path = parse_ctx.path local_path = "path:" + mozpath.relpath(path, self.root) data = self.repo.cat(files=[local_path.encode("utf-8")], rev=self.rev) except Exception: raise paths.ConfigNotFound(parse_ctx.path) parse_ctx.data = toml.loads(data, filename=parse_ctx.path)
def test_relpath(self): self.assertEqual(relpath('foo', 'foo'), '') self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/bar'), '') self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo'), 'bar') self.assertEqual(relpath(self.SEP.join(('foo', 'bar', 'baz')), 'foo'), 'bar/baz') self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/bar/baz'), '..') self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/baz'), '../bar') self.assertEqual(relpath('foo/', 'foo'), '') self.assertEqual(relpath('foo/bar/', 'foo'), 'bar')
def test_relpath(self): self.assertEqual(relpath('foo', 'foo'), '') self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/bar'), '') self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo'), 'bar') self.assertEqual(relpath(self.SEP.join(('foo', 'bar', 'baz')), 'foo'), 'bar/baz') self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/bar/baz'), '..') self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/baz'), '../bar') self.assertEqual(relpath('foo/', 'foo'), '') self.assertEqual(relpath('foo/bar/', 'foo'), 'bar')
def test_no_tests(self): pc = ProjectConfig(None) pc.add_paths({ 'reference': 'some/path/file.ftl', 'l10n': 'some/{locale}/file.ftl', }) files = ProjectFiles(None, [pc]) get_reference_and_tests = util.mirror_reference_and_tests(files, 'tld') ref, tests = get_reference_and_tests('some/path/file.ftl') self.assertEqual(mozpath.relpath(ref, 'tld'), 'some/path/file.ftl') self.assertEqual(tests, set())
def test_no_tests(self): pc = ProjectConfig(None) pc.add_paths({ 'reference': 'some/path/file.ftl', 'l10n': 'some/{locale}/file.ftl', }) files = ProjectFiles(None, [pc]) get_reference_and_tests = util.mirror_reference_and_tests(files, 'tld') ref, tests = get_reference_and_tests('some/path/file.ftl') self.assertEqual(mozpath.relpath(ref, 'tld'), 'some/path/file.ftl') self.assertEqual(tests, set())
def process_config(toml_content): """Process TOML configuration content to match the l10n setup for the reference localization, return target_path and content. The code adjusts basepath, [[paths]], and [[includes]] """ # adjust basepath in content. '.' works in practice, also in theory? new_base = mozpath.relpath(b".", TARGET_PATH) if not new_base: new_base = b"." # relpath to '.' is '', sadly base_line = b'\nbasepath = "%s"' % new_base content1 = re.sub(br"^\s*basepath\s*=\s*.+", base_line, toml_content, flags=re.M) # process [[paths]] start = 0 content2 = b"" for m in re.finditer(br"\[\[\s*paths\s*\]\].+?(?=\[|\Z)", content1, re.M | re.DOTALL): content2 += content1[start:m.start()] path_content = m.group() l10n_line = re.search(br"^\s*l10n\s*=.*$", path_content, flags=re.M).group() # remove variable expansions new_reference = re.sub(br"{\s*\S+\s*}", b"", l10n_line) # make the l10n a reference line new_reference = re.sub(br"^(\s*)l10n(\s*=)", br"\1reference\2", new_reference) content2 += re.sub(br"^\s*reference\s*=.*$", new_reference, path_content, flags=re.M) start = m.end() content2 += content1[start:] start = 0 content3 = b"" for m in re.finditer(br"\[\[\s*includes\s*\]\].+?(?=\[|\Z)", content2, re.M | re.DOTALL): content3 += content2[start:m.start()] include_content = m.group() m_ = re.search(br'^\s*path = "(.+?)"', include_content, flags=re.M) content3 += (include_content[:m_.start(1)] + generate_filename(m_.group(1)) + include_content[m_.end(1):]) start = m.end() content3 += content2[start:] return content3
def test_no_tests(self): pc = ProjectConfig(None) pc.add_environment(l10n_base='l10n_orig') pc.add_paths({ 'reference': 'some/path/file.ftl', 'l10n': '{l10n_base}/{locale}/some/file.ftl', }) pc.set_locales(['gecko'], deep=True) files = ProjectFiles('gecko', [pc]) get_reference_and_tests = util.l10n_base_reference_and_tests(files) ref, tests = get_reference_and_tests('some/path/file.ftl') self.assertEqual(mozpath.relpath(ref, 'l10n_orig/gecko'), 'some/file.ftl') self.assertEqual(tests, set())
def find(self, dir_path): relpath = mozpath.relpath(dir_path, self.root) if relpath.startswith('..'): return None if relpath in ('', '.'): return self segs = mozpath.split(relpath) node = self while segs: seg = segs.pop(0) if seg not in node.dirs: return None node = node.dirs[seg] return node
def _walk(self, base): base = mozpath.normpath(base) local_files = [ mozpath.split(mozpath.relpath(f, base)) for f in self.mocks if f.startswith(base) ] root = MockNode(base) for segs in local_files: node = root for n, seg in enumerate(segs[:-1]): if seg not in node.dirs: node.dirs[seg] = MockNode('/'.join([base] + segs[:n+1])) node = node.dirs[seg] node.files.append(segs[-1]) for tpl in root.walk(): yield tpl
def _walk(self, base): base = mozpath.normpath(base) local_files = [ mozpath.split(mozpath.relpath(f, base)) for f in self.mocks if f.startswith(base) ] root = MockNode(base) for segs in local_files: node = root for n, seg in enumerate(segs[:-1]): if seg not in node.dirs: node.dirs[seg] = MockNode('/'.join([base] + segs[:n+1])) node = node.dirs[seg] node.files.append(segs[-1]) for tpl in root.walk(): yield tpl
def test_no_tests(self): pc = ProjectConfig(None) pc.add_environment(l10n_base='l10n_orig') pc.add_paths({ 'reference': 'some/path/file.ftl', 'l10n': '{l10n_base}/{locale}/some/file.ftl', }) pc.set_locales(['gecko'], deep=True) files = ProjectFiles('gecko', [pc]) get_reference_and_tests = util.l10n_base_reference_and_tests(files) ref, tests = get_reference_and_tests('some/path/file.ftl') self.assertEqual( mozpath.relpath(ref, 'l10n_orig/gecko'), 'some/file.ftl' ) self.assertEqual(tests, set())
def gather(self): fls = files.ProjectFiles("en-US", self.configs) results = [{} for cfg in self.configs] for l10n_path, f, _, tests in fls: if "android-dtd" in tests: # ignore Android native strings continue try: p = parser.getParser(f) except UserWarning: continue p.readFile(f) string_count = len(list(p)) for i, cfg in enumerate(self.configs): l10n_file = paths.File(l10n_path, mozpath.relpath(l10n_path, mozpath.abspath(".")), locale='en-US') if cfg.filter(l10n_file) != "ignore": results[i][l10n_file.file] = string_count return results
def gather_repo(self, repo): basepath = repo.path pc = TOMLParser().parse(mozpath.join(basepath, "l10n.toml")) paths = ["l10n.toml"] + [ mozpath.relpath( m["reference"].pattern.expand(m["reference"].env), basepath ) for m in pc.paths ] self.paths_for_repos[repo.name] = paths branches = repo.branches() self.branches[repo.name] = branches[:] known_revs = self.revs.get(repo.name, {}) for branch_num in range(len(branches)): branch = branches[branch_num] prior_branches = branches[:branch_num] cmd = [ "git", "-C", basepath, "log", "--parents", "--format=%H %ct %P" ] + [ "^" + repo.ref(b) for b in prior_branches ] if branch in known_revs: cmd += ["^" + known_revs[branch]] block_revs = [] elif branch_num == 0: # We haven't seen this repo yet. # Block all known revs in the target from being converted again # in case of repository-level forks. block_revs = self.target.known_revs() cmd += [repo.ref(branch), "--"] + paths out = subprocess.run( cmd, stdout=subprocess.PIPE, encoding="ascii" ).stdout for commit_line in out.splitlines(): segs = commit_line.split() commit = segs.pop(0) if commit in block_revs: continue commit_date = int(segs.pop(0)) self.repos_for_hash[commit].append((repo.name, branch)) self.hashes_for_repo[repo.name].add(commit) self.commit_dates[commit] = max( commit_date, self.commit_dates.get(commit, 0) ) for parent in segs: self.parents[commit].add(parent) self.children[parent].add(commit) if branch in known_revs or branch_num == 0: continue # We don't know this branch yet, and it's a fork. # Find the branch point to the previous branches. for prior_branch in prior_branches: cmd = [ "git", "-C", basepath, "merge-base", repo.ref(branch), repo.ref(prior_branch), ] branch_rev = subprocess.run( cmd, stdout=subprocess.PIPE, encoding="ascii" ).stdout.strip() if not branch_rev: continue # We have a branch revision, find the next child on the # route to the prior branch to add that to. cmd = [ "git", "-C", basepath, "rev-list", "-n", "1", "{}..{}".format(branch_rev, repo.ref(prior_branch)), ] fork_rev = subprocess.run( cmd, stdout=subprocess.PIPE, encoding="ascii" ).stdout.strip() if fork_rev: self.forks[fork_rev].append( (repo.name, branch, branch_rev) )
def compareProjects( project_configs, locales, l10n_base_dir, stat_observer=None, merge_stage=None, clobber_merge=False, quiet=0, ): all_locales = set(locales) comparer = ContentComparer(quiet) observers = comparer.observers for project in project_configs: # disable filter if we're in validation mode if None in locales: filter = None else: filter = project.filter observers.append( Observer( quiet=quiet, filter=filter, )) if not locales: all_locales.update(project.all_locales) for locale in sorted(all_locales): files = paths.ProjectFiles(locale, project_configs, mergebase=merge_stage) if merge_stage is not None: if clobber_merge: mergematchers = set(_m.get('merge') for _m in files.matchers) mergematchers.discard(None) for matcher in mergematchers: clobberdir = matcher.prefix if os.path.exists(clobberdir): shutil.rmtree(clobberdir) print("clobbered " + clobberdir) for l10npath, refpath, mergepath, extra_tests in files: # module and file path are needed for legacy filter.py support module = None fpath = mozpath.relpath(l10npath, l10n_base_dir) for _m in files.matchers: if _m['l10n'].match(l10npath): if _m['module']: # legacy ini support, set module, and resolve # local path against the matcher prefix, # which includes the module module = _m['module'] fpath = mozpath.relpath(l10npath, _m['l10n'].prefix) break reffile = paths.File(refpath, fpath or refpath, module=module) if locale is None: # When validating the reference files, set locale # to a private subtag. This only shows in the output. locale = paths.REFERENCE_LOCALE l10n = paths.File(l10npath, fpath or l10npath, module=module, locale=locale) if not os.path.exists(l10npath): comparer.add(reffile, l10n, mergepath) continue if not os.path.exists(refpath): comparer.remove(reffile, l10n, mergepath) continue comparer.compare(reffile, l10n, mergepath, extra_tests) return observers
def compareProjects( project_configs, l10n_base_dir, stat_observer=None, merge_stage=None, clobber_merge=False, quiet=0, ): locales = set() comparer = ContentComparer(quiet) observers = comparer.observers for project in project_configs: # disable filter if we're in validation mode if None in project.locales: filter = None else: filter = project.filter observers.append(Observer( quiet=quiet, filter=filter, )) locales.update(project.locales) for locale in sorted(locales): files = paths.ProjectFiles(locale, project_configs, mergebase=merge_stage) if merge_stage is not None: if clobber_merge: mergematchers = set(_m.get('merge') for _m in files.matchers) mergematchers.discard(None) for matcher in mergematchers: clobberdir = matcher.prefix if os.path.exists(clobberdir): shutil.rmtree(clobberdir) print("clobbered " + clobberdir) for l10npath, refpath, mergepath, extra_tests in files: # module and file path are needed for legacy filter.py support module = None fpath = mozpath.relpath(l10npath, l10n_base_dir) for _m in files.matchers: if _m['l10n'].match(l10npath): if _m['module']: # legacy ini support, set module, and resolve # local path against the matcher prefix, # which includes the module module = _m['module'] fpath = mozpath.relpath(l10npath, _m['l10n'].prefix) break reffile = paths.File(refpath, fpath or refpath, module=module) if locale is None: # When validating the reference files, set locale # to a private subtag. This only shows in the output. locale = paths.REFERENCE_LOCALE l10n = paths.File(l10npath, fpath or l10npath, module=module, locale=locale) if not os.path.exists(l10npath): comparer.add(reffile, l10n, mergepath) continue if not os.path.exists(refpath): comparer.remove(reffile, l10n, mergepath) continue comparer.compare(reffile, l10n, mergepath, extra_tests) return observers
def __init__(self, mocks, locale, projects, mergebase=None): (super(MockProjectFiles, self) .__init__(locale, projects, mergebase=mergebase)) root = mozpath.commonprefix(mocks) files = [mozpath.relpath(f, root) for f in mocks] self.mocks = MockOS(root, files)
def leaf(self, path): return mozpath.relpath(path, self.root)
def main(): p = argparse.ArgumentParser( description='Validate localizable strings', epilog=epilog, ) p.add_argument('l10n_toml') p.add_argument( '--version', action='version', version='%(prog)s ' + version ) p.add_argument('-W', action='store_true', help='error on warnings') p.add_argument( '--l10n-reference', dest='l10n_reference', metavar='PATH', help='check for conflicts against an l10n-only reference repository ' 'like gecko-strings', ) p.add_argument( '--reference-project', dest='ref_project', metavar='PATH', help='check for conflicts against a reference project like ' 'android-l10n', ) args = p.parse_args() if args.l10n_reference: l10n_base, locale = \ os.path.split(os.path.abspath(args.l10n_reference)) if not locale or not os.path.isdir(args.l10n_reference): p.error('Pass an existing l10n reference') else: l10n_base = '.' locale = None pc = paths.TOMLParser().parse(args.l10n_toml, env={'l10n_base': l10n_base}) if locale: pc.set_locales([locale], deep=True) files = paths.ProjectFiles(locale, [pc]) get_reference_and_tests = default_reference_and_tests if args.l10n_reference: get_reference_and_tests = l10n_base_reference_and_tests(files) elif args.ref_project: get_reference_and_tests = mirror_reference_and_tests( files, args.ref_project ) linter = L10nLinter() results = linter.lint( (f for f, _, _, _ in files.iter_reference() if parser.hasParser(f)), get_reference_and_tests ) rv = 0 if results: rv = 1 if all(r['level'] == 'warning' for r in results) and not args.W: rv = 0 for result in results: print('{} ({}:{}): {}'.format( mozpath.relpath(result['path'], '.'), result.get('lineno', 0), result.get('column', 0), result['message'] )) return rv