def compareProjects(project_configs,
stat_observer=None,
file_stats=False,
merge_stage=None,
clobber_merge=False):
locales = set()
observers = []
for project in project_configs:
observers.append(Observer(filter=project.filter,
file_stats=file_stats))
locales.update(project.locales)
if stat_observer is not None:
stat_observers = [stat_observer]
else:
stat_observers = None
comparer = ContentComparer(observers, stat_observers=stat_observers)
for locale in sorted(locales):
files = paths.ProjectFiles(locale,
project_configs,
mergebase=merge_stage)
root = mozpath.commonprefix([m['l10n'].prefix for m in files.matchers])
if merge_stage is not None:
if clobber_merge:
mergematchers = set(_m.get('merge') for _m in files.matchers)
mergematchers.discard(None)
for matcher in mergematchers:
clobberdir = matcher.prefix
if os.path.exists(clobberdir):
shutil.rmtree(clobberdir)
print "clobbered " + clobberdir
for l10npath, refpath, mergepath, extra_tests in files:
# module and file path are needed for legacy filter.py support
module = None
fpath = mozpath.relpath(l10npath, root)
for _m in files.matchers:
if _m['l10n'].match(l10npath):
if _m['module']:
# legacy ini support, set module, and resolve
# local path against the matcher prefix,
# which includes the module
module = _m['module']
fpath = mozpath.relpath(l10npath, _m['l10n'].prefix)
break
reffile = paths.File(refpath, fpath or refpath, module=module)
l10n = paths.File(l10npath,
fpath or l10npath,
module=module,
locale=locale)
if not os.path.exists(l10npath):
comparer.add(reffile, l10n)
continue
if not os.path.exists(refpath):
comparer.remove(l10n)
continue
comparer.compare(reffile, l10n, mergepath, extra_tests)
return observers
def compareProjects(project_configs, stat_observer=None,
file_stats=False,
merge_stage=None, clobber_merge=False):
locales = set()
observers = []
for project in project_configs:
observers.append(
Observer(filter=project.filter, file_stats=file_stats))
locales.update(project.locales)
if stat_observer is not None:
stat_observers = [stat_observer]
else:
stat_observers = None
comparer = ContentComparer(observers, stat_observers=stat_observers)
for locale in sorted(locales):
files = paths.ProjectFiles(locale, project_configs,
mergebase=merge_stage)
root = mozpath.commonprefix([m['l10n'].prefix for m in files.matchers])
if merge_stage is not None:
if clobber_merge:
mergematchers = set(_m.get('merge') for _m in files.matchers)
mergematchers.discard(None)
for matcher in mergematchers:
clobberdir = matcher.prefix
if os.path.exists(clobberdir):
shutil.rmtree(clobberdir)
print "clobbered " + clobberdir
for l10npath, refpath, mergepath, extra_tests in files:
# module and file path are needed for legacy filter.py support
module = None
fpath = mozpath.relpath(l10npath, root)
for _m in files.matchers:
if _m['l10n'].match(l10npath):
if _m['module']:
# legacy ini support, set module, and resolve
# local path against the matcher prefix,
# which includes the module
module = _m['module']
fpath = mozpath.relpath(l10npath, _m['l10n'].prefix)
break
reffile = paths.File(refpath, fpath or refpath, module=module)
l10n = paths.File(l10npath, fpath or l10npath,
module=module, locale=locale)
if not os.path.exists(l10npath):
comparer.add(reffile, l10n)
continue
if not os.path.exists(refpath):
comparer.remove(l10n)
continue
comparer.compare(reffile, l10n, mergepath, extra_tests)
return observers
def __init__(self, repo, rev, projects):
self.repo = repo
self.ctx = repo[rev]
self.root = repo.root()
self.manifest = None
self.configs_map = {}
# get paths for our TOML files
for p in projects:
all_configpaths = {
mozpath.abspath(c.path).encode("utf-8")
for c in p.configs
}
for refpath in all_configpaths:
local_path = mozpath.relpath(refpath, self.root)
if local_path not in self.ctx:
print("ignoring", refpath)
continue
targetpath = b"/".join((
expand(None, "{l10n_base}", p.environ).encode("utf-8"),
b"en-US",
generate_filename(local_path),
))
self.configs_map[refpath] = targetpath
super(HGFiles, self).__init__("en-US", projects)
for m in self.matchers:
m["l10n"].encoding = "utf-8"
if "reference" in m:
m["reference"].encoding = "utf-8"
if self.exclude:
for m in self.exclude.matchers:
m["l10n"].encoding = "utf-8"
if "reference" in m:
m["reference"].encoding = "utf-8"
def main():
p = argparse.ArgumentParser(
description='Validate localizable strings',
epilog=epilog,
)
p.add_argument('l10n_toml')
p.add_argument('--version',
action='version',
version='%(prog)s ' + version)
p.add_argument('-W', action='store_true', help='error on warnings')
p.add_argument(
'--l10n-reference',
dest='l10n_reference',
metavar='PATH',
help='check for conflicts against an l10n-only reference repository '
'like gecko-strings',
)
p.add_argument(
'--reference-project',
dest='ref_project',
metavar='PATH',
help='check for conflicts against a reference project like '
'android-l10n',
)
args = p.parse_args()
if args.l10n_reference:
l10n_base, locale = \
os.path.split(os.path.abspath(args.l10n_reference))
if not locale or not os.path.isdir(args.l10n_reference):
p.error('Pass an existing l10n reference')
else:
l10n_base = '.'
locale = None
pc = paths.TOMLParser().parse(args.l10n_toml, env={'l10n_base': l10n_base})
if locale:
pc.set_locales([locale], deep=True)
files = paths.ProjectFiles(locale, [pc])
get_reference_and_tests = default_reference_and_tests
if args.l10n_reference:
get_reference_and_tests = l10n_base_reference_and_tests(files)
elif args.ref_project:
get_reference_and_tests = mirror_reference_and_tests(
files, args.ref_project)
linter = L10nLinter()
results = linter.lint(
(f for f, _, _, _ in files.iter_reference() if parser.hasParser(f)),
get_reference_and_tests)
rv = 0
if results:
rv = 1
if all(r['level'] == 'warning' for r in results) and not args.W:
rv = 0
for result in results:
print('{} ({}:{}): {}'.format(mozpath.relpath(result['path'], '.'),
result.get('lineno', 0),
result.get('column', 0),
result['message']))
return rv
def load(self, parse_ctx):
try:
path = parse_ctx.path
local_path = "path:" + mozpath.relpath(path, self.root)
data = self.repo.cat(files=[local_path.encode("utf-8")],
rev=self.rev)
except Exception:
raise paths.ConfigNotFound(parse_ctx.path)
parse_ctx.data = toml.loads(data, filename=parse_ctx.path)
def test_relpath(self):
self.assertEqual(relpath('foo', 'foo'), '')
self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/bar'), '')
self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo'), 'bar')
self.assertEqual(relpath(self.SEP.join(('foo', 'bar', 'baz')), 'foo'),
'bar/baz')
self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/bar/baz'),
'..')
self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/baz'),
'../bar')
self.assertEqual(relpath('foo/', 'foo'), '')
self.assertEqual(relpath('foo/bar/', 'foo'), 'bar')
def test_relpath(self):
self.assertEqual(relpath('foo', 'foo'), '')
self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/bar'), '')
self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo'), 'bar')
self.assertEqual(relpath(self.SEP.join(('foo', 'bar', 'baz')), 'foo'),
'bar/baz')
self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/bar/baz'),
'..')
self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/baz'),
'../bar')
self.assertEqual(relpath('foo/', 'foo'), '')
self.assertEqual(relpath('foo/bar/', 'foo'), 'bar')
def test_no_tests(self):
pc = ProjectConfig(None)
pc.add_paths({
'reference': 'some/path/file.ftl',
'l10n': 'some/{locale}/file.ftl',
})
files = ProjectFiles(None, [pc])
get_reference_and_tests = util.mirror_reference_and_tests(files, 'tld')
ref, tests = get_reference_and_tests('some/path/file.ftl')
self.assertEqual(mozpath.relpath(ref, 'tld'), 'some/path/file.ftl')
self.assertEqual(tests, set())
def test_no_tests(self):
pc = ProjectConfig(None)
pc.add_paths({
'reference': 'some/path/file.ftl',
'l10n': 'some/{locale}/file.ftl',
})
files = ProjectFiles(None, [pc])
get_reference_and_tests = util.mirror_reference_and_tests(files, 'tld')
ref, tests = get_reference_and_tests('some/path/file.ftl')
self.assertEqual(mozpath.relpath(ref, 'tld'), 'some/path/file.ftl')
self.assertEqual(tests, set())
def process_config(toml_content):
"""Process TOML configuration content to match the l10n setup for
the reference localization, return target_path and content.
The code adjusts basepath, [[paths]], and [[includes]]
"""
# adjust basepath in content. '.' works in practice, also in theory?
new_base = mozpath.relpath(b".", TARGET_PATH)
if not new_base:
new_base = b"." # relpath to '.' is '', sadly
base_line = b'\nbasepath = "%s"' % new_base
content1 = re.sub(br"^\s*basepath\s*=\s*.+",
base_line,
toml_content,
flags=re.M)
# process [[paths]]
start = 0
content2 = b""
for m in re.finditer(br"\[\[\s*paths\s*\]\].+?(?=\[|\Z)", content1,
re.M | re.DOTALL):
content2 += content1[start:m.start()]
path_content = m.group()
l10n_line = re.search(br"^\s*l10n\s*=.*$", path_content,
flags=re.M).group()
# remove variable expansions
new_reference = re.sub(br"{\s*\S+\s*}", b"", l10n_line)
# make the l10n a reference line
new_reference = re.sub(br"^(\s*)l10n(\s*=)", br"\1reference\2",
new_reference)
content2 += re.sub(br"^\s*reference\s*=.*$",
new_reference,
path_content,
flags=re.M)
start = m.end()
content2 += content1[start:]
start = 0
content3 = b""
for m in re.finditer(br"\[\[\s*includes\s*\]\].+?(?=\[|\Z)", content2,
re.M | re.DOTALL):
content3 += content2[start:m.start()]
include_content = m.group()
m_ = re.search(br'^\s*path = "(.+?)"', include_content, flags=re.M)
content3 += (include_content[:m_.start(1)] +
generate_filename(m_.group(1)) +
include_content[m_.end(1):])
start = m.end()
content3 += content2[start:]
return content3
def test_no_tests(self):
pc = ProjectConfig(None)
pc.add_environment(l10n_base='l10n_orig')
pc.add_paths({
'reference': 'some/path/file.ftl',
'l10n': '{l10n_base}/{locale}/some/file.ftl',
})
pc.set_locales(['gecko'], deep=True)
files = ProjectFiles('gecko', [pc])
get_reference_and_tests = util.l10n_base_reference_and_tests(files)
ref, tests = get_reference_and_tests('some/path/file.ftl')
self.assertEqual(mozpath.relpath(ref, 'l10n_orig/gecko'),
'some/file.ftl')
self.assertEqual(tests, set())
def find(self, dir_path):
relpath = mozpath.relpath(dir_path, self.root)
if relpath.startswith('..'):
return None
if relpath in ('', '.'):
return self
segs = mozpath.split(relpath)
node = self
while segs:
seg = segs.pop(0)
if seg not in node.dirs:
return None
node = node.dirs[seg]
return node
def _walk(self, base):
base = mozpath.normpath(base)
local_files = [
mozpath.split(mozpath.relpath(f, base))
for f in self.mocks if f.startswith(base)
]
root = MockNode(base)
for segs in local_files:
node = root
for n, seg in enumerate(segs[:-1]):
if seg not in node.dirs:
node.dirs[seg] = MockNode('/'.join([base] + segs[:n+1]))
node = node.dirs[seg]
node.files.append(segs[-1])
for tpl in root.walk():
yield tpl
def _walk(self, base):
base = mozpath.normpath(base)
local_files = [
mozpath.split(mozpath.relpath(f, base))
for f in self.mocks if f.startswith(base)
]
root = MockNode(base)
for segs in local_files:
node = root
for n, seg in enumerate(segs[:-1]):
if seg not in node.dirs:
node.dirs[seg] = MockNode('/'.join([base] + segs[:n+1]))
node = node.dirs[seg]
node.files.append(segs[-1])
for tpl in root.walk():
yield tpl
def test_no_tests(self):
pc = ProjectConfig(None)
pc.add_environment(l10n_base='l10n_orig')
pc.add_paths({
'reference': 'some/path/file.ftl',
'l10n': '{l10n_base}/{locale}/some/file.ftl',
})
pc.set_locales(['gecko'], deep=True)
files = ProjectFiles('gecko', [pc])
get_reference_and_tests = util.l10n_base_reference_and_tests(files)
ref, tests = get_reference_and_tests('some/path/file.ftl')
self.assertEqual(
mozpath.relpath(ref, 'l10n_orig/gecko'),
'some/file.ftl'
)
self.assertEqual(tests, set())
def gather(self):
fls = files.ProjectFiles("en-US", self.configs)
results = [{} for cfg in self.configs]
for l10n_path, f, _, tests in fls:
if "android-dtd" in tests:
# ignore Android native strings
continue
try:
p = parser.getParser(f)
except UserWarning:
continue
p.readFile(f)
string_count = len(list(p))
for i, cfg in enumerate(self.configs):
l10n_file = paths.File(l10n_path,
mozpath.relpath(l10n_path,
mozpath.abspath(".")),
locale='en-US')
if cfg.filter(l10n_file) != "ignore":
results[i][l10n_file.file] = string_count
return results
def gather_repo(self, repo):
basepath = repo.path
pc = TOMLParser().parse(mozpath.join(basepath, "l10n.toml"))
paths = ["l10n.toml"] + [
mozpath.relpath(
m["reference"].pattern.expand(m["reference"].env),
basepath
)
for m in pc.paths
]
self.paths_for_repos[repo.name] = paths
branches = repo.branches()
self.branches[repo.name] = branches[:]
known_revs = self.revs.get(repo.name, {})
for branch_num in range(len(branches)):
branch = branches[branch_num]
prior_branches = branches[:branch_num]
cmd = [
"git", "-C", basepath,
"log",
"--parents",
"--format=%H %ct %P"
] + [
"^" + repo.ref(b) for b in prior_branches
]
if branch in known_revs:
cmd += ["^" + known_revs[branch]]
block_revs = []
elif branch_num == 0:
# We haven't seen this repo yet.
# Block all known revs in the target from being converted again
# in case of repository-level forks.
block_revs = self.target.known_revs()
cmd += [repo.ref(branch), "--"] + paths
out = subprocess.run(
cmd,
stdout=subprocess.PIPE, encoding="ascii"
).stdout
for commit_line in out.splitlines():
segs = commit_line.split()
commit = segs.pop(0)
if commit in block_revs:
continue
commit_date = int(segs.pop(0))
self.repos_for_hash[commit].append((repo.name, branch))
self.hashes_for_repo[repo.name].add(commit)
self.commit_dates[commit] = max(
commit_date, self.commit_dates.get(commit, 0)
)
for parent in segs:
self.parents[commit].add(parent)
self.children[parent].add(commit)
if branch in known_revs or branch_num == 0:
continue
# We don't know this branch yet, and it's a fork.
# Find the branch point to the previous branches.
for prior_branch in prior_branches:
cmd = [
"git",
"-C",
basepath,
"merge-base",
repo.ref(branch),
repo.ref(prior_branch),
]
branch_rev = subprocess.run(
cmd, stdout=subprocess.PIPE, encoding="ascii"
).stdout.strip()
if not branch_rev:
continue
# We have a branch revision, find the next child on the
# route to the prior branch to add that to.
cmd = [
"git",
"-C",
basepath,
"rev-list",
"-n",
"1",
"{}..{}".format(branch_rev, repo.ref(prior_branch)),
]
fork_rev = subprocess.run(
cmd, stdout=subprocess.PIPE, encoding="ascii"
).stdout.strip()
if fork_rev:
self.forks[fork_rev].append(
(repo.name, branch, branch_rev)
)
def compareProjects(
project_configs,
locales,
l10n_base_dir,
stat_observer=None,
merge_stage=None,
clobber_merge=False,
quiet=0,
):
all_locales = set(locales)
comparer = ContentComparer(quiet)
observers = comparer.observers
for project in project_configs:
# disable filter if we're in validation mode
if None in locales:
filter = None
else:
filter = project.filter
observers.append(
Observer(
quiet=quiet,
filter=filter,
))
if not locales:
all_locales.update(project.all_locales)
for locale in sorted(all_locales):
files = paths.ProjectFiles(locale, project_configs,
mergebase=merge_stage)
if merge_stage is not None:
if clobber_merge:
mergematchers = set(_m.get('merge') for _m in files.matchers)
mergematchers.discard(None)
for matcher in mergematchers:
clobberdir = matcher.prefix
if os.path.exists(clobberdir):
shutil.rmtree(clobberdir)
print("clobbered " + clobberdir)
for l10npath, refpath, mergepath, extra_tests in files:
# module and file path are needed for legacy filter.py support
module = None
fpath = mozpath.relpath(l10npath, l10n_base_dir)
for _m in files.matchers:
if _m['l10n'].match(l10npath):
if _m['module']:
# legacy ini support, set module, and resolve
# local path against the matcher prefix,
# which includes the module
module = _m['module']
fpath = mozpath.relpath(l10npath, _m['l10n'].prefix)
break
reffile = paths.File(refpath, fpath or refpath, module=module)
if locale is None:
# When validating the reference files, set locale
# to a private subtag. This only shows in the output.
locale = paths.REFERENCE_LOCALE
l10n = paths.File(l10npath, fpath or l10npath,
module=module, locale=locale)
if not os.path.exists(l10npath):
comparer.add(reffile, l10n, mergepath)
continue
if not os.path.exists(refpath):
comparer.remove(reffile, l10n, mergepath)
continue
comparer.compare(reffile, l10n, mergepath, extra_tests)
return observers
def compareProjects(
project_configs,
l10n_base_dir,
stat_observer=None,
merge_stage=None,
clobber_merge=False,
quiet=0,
):
locales = set()
comparer = ContentComparer(quiet)
observers = comparer.observers
for project in project_configs:
# disable filter if we're in validation mode
if None in project.locales:
filter = None
else:
filter = project.filter
observers.append(Observer(
quiet=quiet,
filter=filter,
))
locales.update(project.locales)
for locale in sorted(locales):
files = paths.ProjectFiles(locale,
project_configs,
mergebase=merge_stage)
if merge_stage is not None:
if clobber_merge:
mergematchers = set(_m.get('merge') for _m in files.matchers)
mergematchers.discard(None)
for matcher in mergematchers:
clobberdir = matcher.prefix
if os.path.exists(clobberdir):
shutil.rmtree(clobberdir)
print("clobbered " + clobberdir)
for l10npath, refpath, mergepath, extra_tests in files:
# module and file path are needed for legacy filter.py support
module = None
fpath = mozpath.relpath(l10npath, l10n_base_dir)
for _m in files.matchers:
if _m['l10n'].match(l10npath):
if _m['module']:
# legacy ini support, set module, and resolve
# local path against the matcher prefix,
# which includes the module
module = _m['module']
fpath = mozpath.relpath(l10npath, _m['l10n'].prefix)
break
reffile = paths.File(refpath, fpath or refpath, module=module)
if locale is None:
# When validating the reference files, set locale
# to a private subtag. This only shows in the output.
locale = paths.REFERENCE_LOCALE
l10n = paths.File(l10npath,
fpath or l10npath,
module=module,
locale=locale)
if not os.path.exists(l10npath):
comparer.add(reffile, l10n, mergepath)
continue
if not os.path.exists(refpath):
comparer.remove(reffile, l10n, mergepath)
continue
comparer.compare(reffile, l10n, mergepath, extra_tests)
return observers
def __init__(self, mocks, locale, projects, mergebase=None):
(super(MockProjectFiles, self)
.__init__(locale, projects, mergebase=mergebase))
root = mozpath.commonprefix(mocks)
files = [mozpath.relpath(f, root) for f in mocks]
self.mocks = MockOS(root, files)
def leaf(self, path):
return mozpath.relpath(path, self.root)
def main():
p = argparse.ArgumentParser(
description='Validate localizable strings',
epilog=epilog,
)
p.add_argument('l10n_toml')
p.add_argument(
'--version', action='version', version='%(prog)s ' + version
)
p.add_argument('-W', action='store_true', help='error on warnings')
p.add_argument(
'--l10n-reference',
dest='l10n_reference',
metavar='PATH',
help='check for conflicts against an l10n-only reference repository '
'like gecko-strings',
)
p.add_argument(
'--reference-project',
dest='ref_project',
metavar='PATH',
help='check for conflicts against a reference project like '
'android-l10n',
)
args = p.parse_args()
if args.l10n_reference:
l10n_base, locale = \
os.path.split(os.path.abspath(args.l10n_reference))
if not locale or not os.path.isdir(args.l10n_reference):
p.error('Pass an existing l10n reference')
else:
l10n_base = '.'
locale = None
pc = paths.TOMLParser().parse(args.l10n_toml, env={'l10n_base': l10n_base})
if locale:
pc.set_locales([locale], deep=True)
files = paths.ProjectFiles(locale, [pc])
get_reference_and_tests = default_reference_and_tests
if args.l10n_reference:
get_reference_and_tests = l10n_base_reference_and_tests(files)
elif args.ref_project:
get_reference_and_tests = mirror_reference_and_tests(
files, args.ref_project
)
linter = L10nLinter()
results = linter.lint(
(f for f, _, _, _ in files.iter_reference() if parser.hasParser(f)),
get_reference_and_tests
)
rv = 0
if results:
rv = 1
if all(r['level'] == 'warning' for r in results) and not args.W:
rv = 0
for result in results:
print('{} ({}:{}): {}'.format(
mozpath.relpath(result['path'], '.'),
result.get('lineno', 0),
result.get('column', 0),
result['message']
))
return rv
