def test_references_wildcard(self, is_dir, glob_mock):
parser = TOMLParser()
def mock_load(ctx):
ctx.data = {
"locales": [],
"paths": [
{
"reference": "some/**/file.ftl",
"l10n": "**/{android_locale}/file.ftl",
},
],
}
parser.load = mock.MagicMock(side_effect=mock_load)
pc = parser.parse("a/basedir/l10n.toml")
is_dir.side_effect = lambda p: not p.endswith("fixed/file.ftl")
glob_mock.return_value = [
pc.root + f for f in [
"/some/other/content",
"/some/other/file.ftl",
"/some/second/deep/file.ftl",
]
]
self.assertListEqual(
process.references(pc, "a/basedir"),
["l10n.toml", "some/other/file.ftl", "some/second/deep/file.ftl"])
def load_configs(lintconfig, root, l10n_base):
"""Load l10n configuration files specified in the linter configuration."""
configs = []
env = {"l10n_base": l10n_base}
for toml in lintconfig["l10n_configs"]:
cfg = TOMLParser().parse(
mozpath.join(root, toml), env=env, ignore_missing_includes=True
)
cfg.set_locales([LOCALE], deep=True)
configs.append(cfg)
return configs
def test_references_file(self, is_dir, glob_mock):
parser = TOMLParser()
def mock_load(ctx):
ctx.data = {
"locales": [],
"paths": [
{
"reference": "some/fixed/file.ftl",
"l10n": "{android_locale}/file.ftl",
},
],
}
parser.load = mock.MagicMock(side_effect=mock_load)
pc = parser.parse("a/basedir/l10n.toml")
is_dir.side_effect = lambda p: not p.endswith("fixed/file.ftl")
glob_mock.side_effect = Exception("not called")
self.assertListEqual(process.references(pc, "a/basedir"),
["l10n.toml", "some/fixed/file.ftl"])
def parsed_configuration(self):
"""Return parsed project configuration file."""
db_project = self.vcs_project.db_project
path = os.path.join(
db_project.source_repository.checkout_path,
self.configuration_file,
)
l10n_base = db_project.translation_repositories()[0].checkout_path
return TOMLParser().parse(path, env={'l10n_base': l10n_base})
def handle(self,
config_paths,
l10n_base_dir,
locales,
merge=None,
defines=None,
unified=False,
full=False,
quiet=0,
validate=False,
clobber=False,
data='text'):
# using nargs multiple times in argparser totally screws things
# up, repair that.
# First files are configs, then the base dir, everything else is
# locales
all_args = config_paths + [l10n_base_dir] + locales
config_paths = []
locales = []
if defines is None:
defines = []
while all_args and not os.path.isdir(all_args[0]):
config_paths.append(all_args.pop(0))
if not config_paths:
self.parser.error('no configuration file given')
for cf in config_paths:
if not os.path.isfile(cf):
self.parser.error('config file %s not found' % cf)
if not all_args:
self.parser.error('l10n-base-dir not found')
l10n_base_dir = all_args.pop(0)
if validate:
# signal validation mode by setting locale list to [None]
locales = [None]
else:
locales.extend(all_args)
# when we compare disabled projects, we set our locales
# on all subconfigs, so deep is True.
locales_deep = full
configs = []
config_env = {}
for define in defines:
var, _, value = define.partition('=')
config_env[var] = value
for config_path in config_paths:
if config_path.endswith('.toml'):
try:
config = TOMLParser.parse(config_path, env=config_env)
except ConfigNotFound as e:
self.parser.exit('config file %s not found' % e.filename)
config.add_global_environment(l10n_base=l10n_base_dir)
if locales:
config.set_locales(locales, deep=locales_deep)
configs.append(config)
else:
app = EnumerateApp(config_path, l10n_base_dir, locales)
configs.append(app.asConfig())
try:
unified_observer = None
if unified:
unified_observer = Observer(quiet=quiet)
observers = compareProjects(configs,
quiet=quiet,
stat_observer=unified_observer,
merge_stage=merge,
clobber_merge=clobber)
except (OSError, IOError) as exc:
print("FAIL: " + str(exc))
self.parser.exit(2)
if unified:
observers = [unified_observer]
rv = 0
for observer in observers:
print(observer.serialize(type=data))
# summary is a dict of lang-summary dicts
# find out if any of our results has errors, return 1 if so
if rv > 0:
continue # we already have errors
for loc, summary in observer.summary.items():
if summary.get('errors', 0) > 0:
rv = 1
# no need to check further summaries, but
# continue to run through observers
break
return rv
def copy_toml(self):
shutil.copy2(self.src_toml, self.dest_toml)
self.dest_config = TOMLParser().parse(self.dest_toml)
def read_configs(self):
parser = TOMLParser()
self.src_config = parser.parse(self.src_toml)
self.dest_config = parser.parse(self.dest_toml)
def handle(
self,
quiet=0, verbose=0,
validate=False,
merge=None,
config_paths=[], l10n_base_dir=None, locales=[],
defines=[],
full=False,
return_zero=False,
clobber=False,
json=None,
):
"""The instance part of the classmethod call.
Using keyword arguments as that is what we need for mach
commands in mozilla-central.
"""
# log as verbose or quiet as we want, warn by default
logging_level = logging.WARNING - (verbose - quiet) * 10
logging.basicConfig()
logging.getLogger().setLevel(logging_level)
config_paths, l10n_base_dir, locales = self.extract_positionals(
validate=validate,
config_paths=config_paths,
l10n_base_dir=l10n_base_dir,
locales=locales,
)
# when we compare disabled projects, we set our locales
# on all subconfigs, so deep is True.
locales_deep = full
configs = []
config_env = {
'l10n_base': l10n_base_dir
}
for define in defines:
var, _, value = define.partition('=')
config_env[var] = value
for config_path in config_paths:
if config_path.endswith('.toml'):
try:
config = TOMLParser().parse(config_path, env=config_env)
except ConfigNotFound as e:
self.parser.exit('config file %s not found' % e.filename)
if locales_deep:
if not locales:
# no explicit locales given, force all locales
config.set_locales(config.all_locales, deep=True)
else:
config.set_locales(locales, deep=True)
configs.append(config)
else:
app = EnumerateApp(config_path, l10n_base_dir)
configs.append(app.asConfig())
try:
observers = compareProjects(
configs,
locales,
l10n_base_dir,
quiet=quiet,
merge_stage=merge, clobber_merge=clobber)
except (OSError, IOError) as exc:
print("FAIL: " + str(exc))
self.parser.exit(2)
if json is None or json != '-':
details = observers.serializeDetails()
if details:
print(details)
if len(configs) > 1:
if details:
print('')
print("Summaries for")
for config_path in config_paths:
print(" " + config_path)
print(" and the union of these, counting each string once")
print(observers.serializeSummaries())
if json is not None:
data = [observer.toJSON() for observer in observers]
stdout = json == '-'
indent = 1 if stdout else None
fh = sys.stdout if stdout else open(json, 'w')
json_dump(data, fh, sort_keys=True, indent=indent)
if stdout:
fh.write('\n')
fh.close()
rv = 1 if not return_zero and observers.error else 0
return rv
def parsed_configuration(self):
"""Return parsed project configuration file."""
return TOMLParser().parse(
self.configuration_path,
env={'l10n_base': self.l10n_base},
)
def handle(
self,
quiet=0,
verbose=0,
validate=False,
merge=None,
config_paths=[],
l10n_base_dir=None,
locales=[],
defines=[],
full=False,
return_zero=False,
clobber=False,
json=None,
):
"""The instance part of the classmethod call.
Using keyword arguments as that is what we need for mach
commands in mozilla-central.
"""
# log as verbose or quiet as we want, warn by default
logging_level = logging.WARNING - (verbose - quiet) * 10
logging.basicConfig()
logging.getLogger().setLevel(logging_level)
config_paths, l10n_base_dir, locales = self.extract_positionals(
validate=validate,
config_paths=config_paths,
l10n_base_dir=l10n_base_dir,
locales=locales,
)
# when we compare disabled projects, we set our locales
# on all subconfigs, so deep is True.
locales_deep = full
configs = []
config_env = {'l10n_base': l10n_base_dir}
for define in defines:
var, _, value = define.partition('=')
config_env[var] = value
for config_path in config_paths:
if config_path.endswith('.toml'):
try:
config = TOMLParser().parse(config_path, env=config_env)
except ConfigNotFound as e:
self.parser.exit('config file %s not found' % e.filename)
if locales_deep:
if not locales:
# no explicit locales given, force all locales
config.set_locales(config.all_locales, deep=True)
else:
config.set_locales(locales, deep=True)
configs.append(config)
else:
app = EnumerateApp(config_path, l10n_base_dir)
configs.append(app.asConfig())
try:
observers = compareProjects(configs,
locales,
l10n_base_dir,
quiet=quiet,
merge_stage=merge,
clobber_merge=clobber)
except (OSError, IOError) as exc:
print("FAIL: " + str(exc))
self.parser.exit(2)
if json is None or json != '-':
details = observers.serializeDetails()
if details:
print(details)
if len(configs) > 1:
if details:
print('')
print("Summaries for")
for config_path in config_paths:
print(" " + config_path)
print(" and the union of these, counting each string once")
print(observers.serializeSummaries())
if json is not None:
data = [observer.toJSON() for observer in observers]
stdout = json == '-'
indent = 1 if stdout else None
fh = sys.stdout if stdout else open(json, 'w')
json_dump(data, fh, sort_keys=True, indent=indent)
if stdout:
fh.write('\n')
fh.close()
rv = 1 if not return_zero and observers.error else 0
return rv
def gather_repo(self, repo):
basepath = repo.path
pc = TOMLParser().parse(mozpath.join(basepath, "l10n.toml"))
paths = ["l10n.toml"] + [
mozpath.relpath(
m["reference"].pattern.expand(m["reference"].env),
basepath
)
for m in pc.paths
]
self.paths_for_repos[repo.name] = paths
branches = repo.branches()
self.branches[repo.name] = branches[:]
known_revs = self.revs.get(repo.name, {})
for branch_num in range(len(branches)):
branch = branches[branch_num]
prior_branches = branches[:branch_num]
cmd = [
"git", "-C", basepath,
"log",
"--parents",
"--format=%H %ct %P"
] + [
"^" + repo.ref(b) for b in prior_branches
]
if branch in known_revs:
cmd += ["^" + known_revs[branch]]
block_revs = []
elif branch_num == 0:
# We haven't seen this repo yet.
# Block all known revs in the target from being converted again
# in case of repository-level forks.
block_revs = self.target.known_revs()
cmd += [repo.ref(branch), "--"] + paths
out = subprocess.run(
cmd,
stdout=subprocess.PIPE, encoding="ascii"
).stdout
for commit_line in out.splitlines():
segs = commit_line.split()
commit = segs.pop(0)
if commit in block_revs:
continue
commit_date = int(segs.pop(0))
self.repos_for_hash[commit].append((repo.name, branch))
self.hashes_for_repo[repo.name].add(commit)
self.commit_dates[commit] = max(
commit_date, self.commit_dates.get(commit, 0)
)
for parent in segs:
self.parents[commit].add(parent)
self.children[parent].add(commit)
if branch in known_revs or branch_num == 0:
continue
# We don't know this branch yet, and it's a fork.
# Find the branch point to the previous branches.
for prior_branch in prior_branches:
cmd = [
"git",
"-C",
basepath,
"merge-base",
repo.ref(branch),
repo.ref(prior_branch),
]
branch_rev = subprocess.run(
cmd, stdout=subprocess.PIPE, encoding="ascii"
).stdout.strip()
if not branch_rev:
continue
# We have a branch revision, find the next child on the
# route to the prior branch to add that to.
cmd = [
"git",
"-C",
basepath,
"rev-list",
"-n",
"1",
"{}..{}".format(branch_rev, repo.ref(prior_branch)),
]
fork_rev = subprocess.run(
cmd, stdout=subprocess.PIPE, encoding="ascii"
).stdout.strip()
if fork_rev:
self.forks[fork_rev].append(
(repo.name, branch, branch_rev)
)
def handle(self,
config_paths,
l10n_base_dir,
locales,
merge=None,
defines=None,
unified=False,
full=False,
quiet=0,
clobber=False,
data='text'):
# using nargs multiple times in argparser totally screws things
# up, repair that.
# First files are configs, then the base dir, everything else is
# locales
all_args = config_paths + [l10n_base_dir] + locales
config_paths = []
locales = []
if defines is None:
defines = []
while all_args and not os.path.isdir(all_args[0]):
config_paths.append(all_args.pop(0))
if not config_paths:
self.parser.error('no configuration file given')
for cf in config_paths:
if not os.path.isfile(cf):
self.parser.error('config file %s not found' % cf)
if not all_args:
self.parser.error('l10n-base-dir not found')
l10n_base_dir = all_args.pop(0)
locales.extend(all_args)
# when we compare disabled projects, we set our locales
# on all subconfigs, so deep is True.
locales_deep = full
configs = []
config_env = {}
for define in defines:
var, _, value = define.partition('=')
config_env[var] = value
for config_path in config_paths:
if config_path.endswith('.toml'):
try:
config = TOMLParser.parse(config_path, env=config_env)
except ConfigNotFound as e:
self.parser.exit('config file %s not found' % e.filename)
config.add_global_environment(l10n_base=l10n_base_dir)
if locales:
config.set_locales(locales, deep=locales_deep)
configs.append(config)
else:
app = EnumerateApp(config_path, l10n_base_dir, locales)
configs.append(app.asConfig())
try:
unified_observer = None
if unified:
unified_observer = Observer(quiet=quiet)
observers = compareProjects(configs,
quiet=quiet,
stat_observer=unified_observer,
merge_stage=merge,
clobber_merge=clobber)
except (OSError, IOError), exc:
print "FAIL: " + str(exc)
self.parser.exit(2)
def get_config(toml_path):
c = TOMLParser().parse(toml_path)
return c
def compare(browser_toml, l10n_base_dir):
config_env = {"l10n_base": l10n_base_dir}
configs = [TOMLParser().parse(browser_toml, env=config_env)]
return compareProjects(configs, ["ja"], l10n_base_dir).toJSON()
