def compareProjects(project_configs,
stat_observer=None,
file_stats=False,
merge_stage=None,
clobber_merge=False):
locales = set()
observers = []
for project in project_configs:
observers.append(Observer(filter=project.filter,
file_stats=file_stats))
locales.update(project.locales)
if stat_observer is not None:
stat_observers = [stat_observer]
else:
stat_observers = None
comparer = ContentComparer(observers, stat_observers=stat_observers)
for locale in sorted(locales):
files = paths.ProjectFiles(locale,
project_configs,
mergebase=merge_stage)
root = mozpath.commonprefix([m['l10n'].prefix for m in files.matchers])
if merge_stage is not None:
if clobber_merge:
mergematchers = set(_m.get('merge') for _m in files.matchers)
mergematchers.discard(None)
for matcher in mergematchers:
clobberdir = matcher.prefix
if os.path.exists(clobberdir):
shutil.rmtree(clobberdir)
print "clobbered " + clobberdir
for l10npath, refpath, mergepath, extra_tests in files:
# module and file path are needed for legacy filter.py support
module = None
fpath = mozpath.relpath(l10npath, root)
for _m in files.matchers:
if _m['l10n'].match(l10npath):
if _m['module']:
# legacy ini support, set module, and resolve
# local path against the matcher prefix,
# which includes the module
module = _m['module']
fpath = mozpath.relpath(l10npath, _m['l10n'].prefix)
break
reffile = paths.File(refpath, fpath or refpath, module=module)
l10n = paths.File(l10npath,
fpath or l10npath,
module=module,
locale=locale)
if not os.path.exists(l10npath):
comparer.add(reffile, l10n)
continue
if not os.path.exists(refpath):
comparer.remove(l10n)
continue
comparer.compare(reffile, l10n, mergepath, extra_tests)
return observers
def test_simple(self):
obs = compare.Observer()
f = paths.File('/some/real/sub/path', 'sub/path', locale='de')
obs.notify('missingEntity', f, 'one')
obs.notify('missingEntity', f, 'two')
obs.updateStats(f, {'missing': 15})
self.assertDictEqual(
obs.toJSON(), {
'summary': {
'de': {
'missing': 15
}
},
'details': {
'de/sub/path': [{
'missingEntity': 'one'
}, {
'missingEntity': 'two'
}]
}
})
clone = loads(dumps(obs))
self.assertDictEqual(clone.summary, obs.summary)
self.assertDictEqual(clone.details.toJSON(), obs.details.toJSON())
self.assertIsNone(clone.file_stats)
def test_simple(self):
obs = compare.Observer()
f = paths.File('/some/real/sub/path', 'de/sub/path', locale='de')
obs.notify('missingEntity', f, 'one')
obs.notify('missingEntity', f, 'two')
obs.updateStats(f, {'missing': 15})
self.assertDictEqual(obs.toJSON(), {
'summary': {
'de': {
'errors': 0,
'warnings': 0,
'missing': 15,
'missing_w': 0,
'report': 0,
'obsolete': 0,
'changed': 0,
'changed_w': 0,
'unchanged': 0,
'unchanged_w': 0,
'keys': 0,
}
},
'details': {
'de/sub/path':
[{'missingEntity': 'one'},
{'missingEntity': 'two'}]
}
})
def test_module(self):
obs = compare.Observer()
f = paths.File('/some/real/sub/path',
'path',
module='sub',
locale='de')
obs.notify('missingEntity', f, 'one')
obs.notify('obsoleteEntity', f, 'bar')
obs.notify('missingEntity', f, 'two')
obs.updateStats(f, {'missing': 15})
self.assertDictEqual(
obs.toJSON(), {
'summary': {
'de': {
'missing': 15
}
},
'details': {
'de/sub/path': [
{
'missingEntity': 'one'
},
{
'obsoleteEntity': 'bar'
},
{
'missingEntity': 'two'
},
]
}
})
def test_module(self):
obs = compare.Observer(file_stats=True)
f = paths.File('/some/real/sub/path',
'path',
module='sub',
locale='de')
obs.notify('missingEntity', f, 'one')
obs.notify('obsoleteEntity', f, 'bar')
obs.notify('missingEntity', f, 'two')
obs.updateStats(f, {'missing': 15})
self.assertDictEqual(
obs.toJSON(), {
'summary': {
'de': {
'missing': 15
}
},
'details': {
'de/sub/path': [
{
'missingEntity': 'one'
},
{
'obsoleteEntity': 'bar'
},
{
'missingEntity': 'two'
},
]
}
})
self.assertDictEqual(obs.file_stats,
{'de': {
'sub/path': {
'missing': 15
}
}})
self.assertEqual(
obs.serialize(), '''\
de/sub/path
+one
-bar
+two
de:
missing: 15
0% of entries changed''')
clone = loads(dumps(obs))
self.assertDictEqual(clone.summary, obs.summary)
self.assertDictEqual(clone.details.toJSON(), obs.details.toJSON())
self.assertDictEqual(clone.file_stats, obs.file_stats)
def gather(self):
fls = files.ProjectFiles("en-US", self.configs)
results = [{} for cfg in self.configs]
for l10n_path, f, _, tests in fls:
if "android-dtd" in tests:
# ignore Android native strings
continue
try:
p = parser.getParser(f)
except UserWarning:
continue
p.readFile(f)
string_count = len(list(p))
for i, cfg in enumerate(self.configs):
l10n_file = paths.File(l10n_path,
mozpath.relpath(l10n_path,
mozpath.abspath(".")),
locale='en-US')
if cfg.filter(l10n_file) != "ignore":
results[i][l10n_file.file] = string_count
return results
def compareProjects(
project_configs,
l10n_base_dir,
stat_observer=None,
merge_stage=None,
clobber_merge=False,
quiet=0,
):
locales = set()
comparer = ContentComparer(quiet)
observers = comparer.observers
for project in project_configs:
# disable filter if we're in validation mode
if None in project.locales:
filter = None
else:
filter = project.filter
observers.append(Observer(
quiet=quiet,
filter=filter,
))
locales.update(project.locales)
for locale in sorted(locales):
files = paths.ProjectFiles(locale,
project_configs,
mergebase=merge_stage)
if merge_stage is not None:
if clobber_merge:
mergematchers = set(_m.get('merge') for _m in files.matchers)
mergematchers.discard(None)
for matcher in mergematchers:
clobberdir = matcher.prefix
if os.path.exists(clobberdir):
shutil.rmtree(clobberdir)
print("clobbered " + clobberdir)
for l10npath, refpath, mergepath, extra_tests in files:
# module and file path are needed for legacy filter.py support
module = None
fpath = mozpath.relpath(l10npath, l10n_base_dir)
for _m in files.matchers:
if _m['l10n'].match(l10npath):
if _m['module']:
# legacy ini support, set module, and resolve
# local path against the matcher prefix,
# which includes the module
module = _m['module']
fpath = mozpath.relpath(l10npath, _m['l10n'].prefix)
break
reffile = paths.File(refpath, fpath or refpath, module=module)
if locale is None:
# When validating the reference files, set locale
# to a private subtag. This only shows in the output.
locale = paths.REFERENCE_LOCALE
l10n = paths.File(l10npath,
fpath or l10npath,
module=module,
locale=locale)
if not os.path.exists(l10npath):
comparer.add(reffile, l10n, mergepath)
continue
if not os.path.exists(refpath):
comparer.remove(reffile, l10n, mergepath)
continue
comparer.compare(reffile, l10n, mergepath, extra_tests)
return observers
