def resource_paths_with_pc(self):
"""
List of absolute paths for all supported source resources
as specified through project configuration.
"""
path = self.source_directory_path
project_files = ProjectFiles(None,
[self.configuration.parsed_configuration])
for root, dirnames, filenames in scandir.walk(path):
if is_hidden(root):
continue
for filename in filenames:
absolute_path = os.path.join(root, filename)
if project_files.match(absolute_path):
yield absolute_path
def test_reference(self, _walk, _isfile):
pontoon_config, vendor_config, files = self._list(None, _walk, _isfile)
pontoon_files = ProjectFiles(None, [pontoon_config])
vendor_files = ProjectFiles(None, [vendor_config])
self.assertListEqual(files, [
'firefox/feature.ftl',
'firefox/home.ftl',
'mozorg/mission.ftl',
])
ref_path = self.path('/en/firefox/feature.ftl')
self.assertIsNotNone(pontoon_files.match(ref_path))
self.assertIsNotNone(vendor_files.match(ref_path))
ref_path = self.path('/en/firefox/home.ftl')
self.assertIsNotNone(pontoon_files.match(ref_path))
self.assertIsNotNone(vendor_files.match(ref_path))
ref_path = self.path('/en/mozorg/mission.ftl')
self.assertIsNotNone(pontoon_files.match(ref_path))
self.assertIsNone(vendor_files.match(ref_path))
def locale_resources(self, locale):
"""
Return a list of Resource instances, which need to be enabled for the
given locale.
"""
resources = []
project_files = ProjectFiles(locale.code, [self.parsed_configuration])
for resource in self.vcs_project.db_project.resources.all():
absolute_resource_path = os.path.join(
self.vcs_project.source_directory_path,
resource.path,
)
if project_files.match(absolute_resource_path):
resources.append(resource)
return resources
def lint(paths, lintconfig, **lintargs):
l10n_base = mb_util.get_state_dir()
root = lintargs["root"]
exclude = lintconfig.get("exclude")
extensions = lintconfig.get("extensions")
# Load l10n.toml configs
l10nconfigs = load_configs(lintconfig, root, l10n_base)
# Check include paths in l10n.yml if it's in our given paths
# Only the l10n.yml will show up here, but if the l10n.toml files
# change, we also get the l10n.yml as the toml files are listed as
# support files.
if lintconfig["path"] in paths:
results = validate_linter_includes(lintconfig, l10nconfigs, lintargs)
paths.remove(lintconfig["path"])
else:
results = []
all_files = []
for p in paths:
fp = pathutils.FilterPath(p)
if fp.isdir:
for _, fileobj in fp.finder:
all_files.append(fileobj.path)
if fp.isfile:
all_files.append(p)
# Filter again, our directories might have picked up files the
# explicitly excluded in the l10n.yml configuration.
# `browser/locales/en-US/firefox-l10n.js` is a good example.
all_files, _ = pathutils.filterpaths(
lintargs["root"],
all_files,
lintconfig["include"],
exclude=exclude,
extensions=extensions,
)
# These should be excluded in l10n.yml
skips = {p for p in all_files if not parser.hasParser(p)}
results.extend(
result.from_config(
lintconfig,
level="warning",
path=path,
message="file format not supported in compare-locales",
)
for path in skips
)
all_files = [p for p in all_files if p not in skips]
files = ProjectFiles(LOCALE, l10nconfigs)
get_reference_and_tests = l10n_base_reference_and_tests(files)
linter = MozL10nLinter(lintconfig)
results += linter.lint(all_files, get_reference_and_tests)
return results
def get_or_set_project_files(self, locale_code):
"""
Get or set project files for the given locale code. This approach
allows us to cache the files for later use.
Also, make sure that the requested locale_code is available in the
configuration file.
"""
if (locale_code is not None
and locale_code not in self.parsed_configuration.all_locales):
self.add_locale(locale_code)
return self.project_files.setdefault(
locale_code,
ProjectFiles(locale_code, [self.parsed_configuration]),
)
def test_src_app(self):
'Test parsing a App in source setup'
# move toolkit to toplevel
shutil.move(mozpath.join(self.stage, 'comm', 'mozilla'), self.stage)
app = EnumerateSourceTreeApp(
mozpath.join(self.stage, 'comm', 'mail', 'locales', 'l10n.ini'),
self.stage,
mozpath.join(self.stage, 'l10n-central'),
{
'mozilla-central': mozpath.join(self.stage, 'mozilla')
}
)
self.assertListEqual(app.config.allLocales(), ['af', 'de', 'fr'])
self.assertEqual(len(app.config.children), 1)
projectconfig = app.asConfig()
self.assertListEqual(projectconfig.locales, ['af', 'de', 'fr'])
files = ProjectFiles('de', [projectconfig])
files = list(files)
self.assertEqual(len(files), 3)
l10nfile, reffile, mergefile, test = files[0]
self.assertListEqual(mozpath.split(l10nfile)[-3:],
['de', 'mail', 'mail.ftl'])
self.assertListEqual(mozpath.split(reffile)[-4:],
['mail', 'locales', 'en-US', 'mail.ftl'])
self.assertIsNone(mergefile)
self.assertSetEqual(test, set())
l10nfile, reffile, mergefile, test = files[1]
self.assertListEqual(mozpath.split(l10nfile)[-3:],
['de', 'toolkit', 'localized.ftl'])
self.assertListEqual(
mozpath.split(reffile)[-5:],
['mozilla', 'toolkit',
'locales', 'en-US', 'localized.ftl'])
self.assertIsNone(mergefile)
self.assertSetEqual(test, set())
l10nfile, reffile, mergefile, test = files[2]
self.assertListEqual(mozpath.split(l10nfile)[-3:],
['de', 'toolkit', 'platform.ftl'])
self.assertListEqual(
mozpath.split(reffile)[-5:],
['mozilla', 'toolkit', 'locales', 'en-US', 'platform.ftl'])
self.assertIsNone(mergefile)
self.assertSetEqual(test, set())
def test_it(self, _walk, _isfile):
# all pages translated, but split between vendor and community
pontoon_config, vendor_config, files = self._list('it', _walk, _isfile)
pontoon_files = ProjectFiles('it', [pontoon_config])
vendor_files = ProjectFiles('it', [vendor_config])
self.assertListEqual(files, [
'firefox/feature.ftl',
'firefox/home.ftl',
'mozorg/mission.ftl',
])
l10n_path = self.path('/it/firefox/home.ftl')
ref_path = self.path('/en/firefox/home.ftl')
file = File(l10n_path, 'it/firefox/home.ftl', locale='it')
self.assertEqual(pontoon_config.filter(file), 'ignore')
self.assertEqual(vendor_config.filter(file), 'error')
self.assertIsNone(pontoon_files.match(l10n_path))
self.assertIsNone(pontoon_files.match(ref_path))
self.assertIsNotNone(vendor_files.match(l10n_path))
self.assertIsNotNone(vendor_files.match(ref_path))
def test_gd(self, _walk, _isfile):
# only community localization
pontoon_config, vendor_config, files = self._list('gd', _walk, _isfile)
pontoon_files = ProjectFiles('gd', [pontoon_config])
vendor_files = ProjectFiles('gd', [vendor_config])
self.assertListEqual(files, [
'firefox/feature.ftl',
'firefox/home.ftl',
'mozorg/mission.ftl',
])
l10n_path = self.path('/gd/firefox/home.ftl')
ref_path = self.path('/en/firefox/home.ftl')
self.assertEqual(
pontoon_config.filter(
File(l10n_path, 'gd/firefox/home.ftl', locale='gd')), 'error')
self.assertIsNotNone(pontoon_files.match(l10n_path))
self.assertIsNotNone(pontoon_files.match(ref_path))
self.assertIsNone(vendor_files.match(l10n_path))
self.assertIsNone(vendor_files.match(ref_path))
def test_app(self):
'Test parsing a App'
app = EnumerateApp(
mozpath.join(self.stage, 'comm', 'mail', 'locales', 'l10n.ini'),
mozpath.join(self.stage, 'l10n-central'))
self.assertListEqual(app.locales, ['af', 'de', 'fr'])
self.assertEqual(len(app.config.children), 1)
projectconfig = app.asConfig()
self.assertListEqual(projectconfig.locales, ['af', 'de', 'fr'])
files = ProjectFiles('de', [projectconfig])
files = list(files)
self.assertEqual(len(files), 3)
l10nfile, reffile, mergefile, test = files[0]
self.assertListEqual(
mozpath.split(l10nfile)[-3:], ['de', 'mail', 'mail.ftl'])
self.assertListEqual(
mozpath.split(reffile)[-4:],
['mail', 'locales', 'en-US', 'mail.ftl'])
self.assertIsNone(mergefile)
self.assertSetEqual(test, set())
l10nfile, reffile, mergefile, test = files[1]
self.assertListEqual(
mozpath.split(l10nfile)[-3:], ['de', 'toolkit', 'localized.ftl'])
self.assertListEqual(
mozpath.split(reffile)[-6:], [
'comm', 'mozilla', 'toolkit', 'locales', 'en-US',
'localized.ftl'
])
self.assertIsNone(mergefile)
self.assertSetEqual(test, set())
l10nfile, reffile, mergefile, test = files[2]
self.assertListEqual(
mozpath.split(l10nfile)[-3:], ['de', 'toolkit', 'platform.ftl'])
self.assertListEqual(
mozpath.split(reffile)[-6:],
['comm', 'mozilla', 'toolkit', 'locales', 'en-US', 'platform.ftl'])
self.assertIsNone(mergefile)
self.assertSetEqual(test, set())
def xliff(self, locale):
files = ProjectFiles(locale, [self.config])
root = ET.Element(f"{{{xmlns}}}xliff", {
"version": "2.0",
"srcLang": "en",
"trgLang": locale,
})
root.text = "\n "
root.tail = "\n"
previous = None
for l10n, ref, _, _ in files:
if previous is not None:
previous.tail += " "
ref_id = os.path.relpath(ref, self.config.root)
ref_id = ref_id.replace("/", "-")
file_elem = ET.SubElement(root, f"{{{xmlns}}}file", {"id": ref_id})
file_elem.tail = "\n"
file_elem.text = "\n "
self.file(file_elem, ref, l10n, locale)
previous = file_elem
return ET.tostring(root)
def test_de(self, _walk, _isfile):
# home.ftl excluded completely by configs-special-templates.toml
# firefox/* only in vendor
pontoon_config, vendor_config, files = self._list('de', _walk, _isfile)
pontoon_files = ProjectFiles('de', [pontoon_config])
vendor_files = ProjectFiles('de', [vendor_config])
self.assertListEqual(
files,
[
'firefox/feature.ftl',
# 'firefox/home.ftl',
'mozorg/mission.ftl',
])
l10n_path = self.path('/de/firefox/feature.ftl')
ref_path = self.path('/en/firefox/feature.ftl')
self.assertEqual(
pontoon_config.filter(
File(l10n_path, 'de/firefox/feature.ftl', locale='de')),
'ignore')
self.assertIsNone(pontoon_files.match(l10n_path))
self.assertIsNone(pontoon_files.match(ref_path))
self.assertIsNotNone(vendor_files.match(l10n_path))
self.assertIsNotNone(vendor_files.match(ref_path))
l10n_path = self.path('/de/firefox/home.ftl')
ref_path = self.path('/en/firefox/home.ftl')
self.assertEqual(
pontoon_config.filter(
File(l10n_path, 'de/firefox/home.ftl', locale='de')), 'ignore')
self.assertIsNone(pontoon_files.match(l10n_path))
self.assertIsNone(pontoon_files.match(ref_path))
self.assertIsNone(vendor_files.match(l10n_path))
self.assertIsNone(vendor_files.match(ref_path))
l10n_path = self.path('/de/mozorg/mission.ftl')
ref_path = self.path('/en/mozorg/mission.ftl')
self.assertEqual(
pontoon_config.filter(
File(l10n_path, 'de/mozorg/mission.ftl', locale='de')),
'error')
self.assertIsNotNone(pontoon_files.match(l10n_path))
self.assertIsNotNone(pontoon_files.match(ref_path))
self.assertIsNone(vendor_files.match(l10n_path))
self.assertIsNone(vendor_files.match(ref_path))
def _list(self, locale, _walk, _isfile):
parser = MockTOMLParser({
"pontoon.toml":
'''\
basepath = "."
[[includes]]
path = "configs-pontoon.toml"
[[excludes]]
path = "configs-vendor.toml"
[[excludes]]
path = "configs-special-templates.toml"
''',
"vendor.toml":
'''\
basepath = "."
[[includes]]
path = "configs-vendor.toml"
[[excludes]]
path = "configs-special-templates.toml"
''',
"configs-pontoon.toml":
'''\
basepath = "."
locales = [
"de",
"gd",
"it",
]
[[paths]]
reference = "en/**/*.ftl"
l10n = "{locale}/**/*.ftl"
''',
"configs-vendor.toml":
'''\
basepath = "."
locales = [
"de",
"it",
]
[[paths]]
reference = "en/firefox/*.ftl"
l10n = "{locale}/firefox/*.ftl"
''',
"configs-special-templates.toml":
'''\
basepath = "."
[[paths]]
reference = "en/firefox/home.ftl"
l10n = "{locale}/firefox/home.ftl"
locales = [
"de",
"fr",
]
[[paths]]
reference = "en/firefox/pagina.ftl"
l10n = "{locale}/firefox/pagina.ftl"
locales = [
"gd",
]
''',
})
pontoon = parser.parse(self.path('/pontoon.toml'))
vendor = parser.parse(self.path('/vendor.toml'))
pc = ProjectFiles(locale, [pontoon, vendor])
mock_files = [
'{}/{}/{}'.format(locale, dir, f)
for locale in ('de', 'en', 'gd', 'it') for dir, files in (
('firefox', ('home.ftl', 'feature.ftl')),
('mozorg', ('mission.ftl', )),
) for f in files
]
os_ = MockOS(self.root, mock_files)
_isfile.side_effect = os_.isfile
_walk.side_effect = os_.walk
local_files = [self.leaf(p).split('/', 1)[1] for p, _, _, _ in pc]
return pontoon, vendor, local_files
