def compareProjects(project_configs, stat_observer=None, file_stats=False, merge_stage=None, clobber_merge=False): locales = set() observers = [] for project in project_configs: observers.append(Observer(filter=project.filter, file_stats=file_stats)) locales.update(project.locales) if stat_observer is not None: stat_observers = [stat_observer] else: stat_observers = None comparer = ContentComparer(observers, stat_observers=stat_observers) for locale in sorted(locales): files = paths.ProjectFiles(locale, project_configs, mergebase=merge_stage) root = mozpath.commonprefix([m['l10n'].prefix for m in files.matchers]) if merge_stage is not None: if clobber_merge: mergematchers = set(_m.get('merge') for _m in files.matchers) mergematchers.discard(None) for matcher in mergematchers: clobberdir = matcher.prefix if os.path.exists(clobberdir): shutil.rmtree(clobberdir) print "clobbered " + clobberdir for l10npath, refpath, mergepath, extra_tests in files: # module and file path are needed for legacy filter.py support module = None fpath = mozpath.relpath(l10npath, root) for _m in files.matchers: if _m['l10n'].match(l10npath): if _m['module']: # legacy ini support, set module, and resolve # local path against the matcher prefix, # which includes the module module = _m['module'] fpath = mozpath.relpath(l10npath, _m['l10n'].prefix) break reffile = paths.File(refpath, fpath or refpath, module=module) l10n = paths.File(l10npath, fpath or l10npath, module=module, locale=locale) if not os.path.exists(l10npath): comparer.add(reffile, l10n) continue if not os.path.exists(refpath): comparer.remove(l10n) continue comparer.compare(reffile, l10n, mergepath, extra_tests) return observers
def test_simple(self): obs = compare.Observer() f = paths.File('/some/real/sub/path', 'sub/path', locale='de') obs.notify('missingEntity', f, 'one') obs.notify('missingEntity', f, 'two') obs.updateStats(f, {'missing': 15}) self.assertDictEqual( obs.toJSON(), { 'summary': { 'de': { 'missing': 15 } }, 'details': { 'de/sub/path': [{ 'missingEntity': 'one' }, { 'missingEntity': 'two' }] } }) clone = loads(dumps(obs)) self.assertDictEqual(clone.summary, obs.summary) self.assertDictEqual(clone.details.toJSON(), obs.details.toJSON()) self.assertIsNone(clone.file_stats)
def test_simple(self): obs = compare.Observer() f = paths.File('/some/real/sub/path', 'de/sub/path', locale='de') obs.notify('missingEntity', f, 'one') obs.notify('missingEntity', f, 'two') obs.updateStats(f, {'missing': 15}) self.assertDictEqual(obs.toJSON(), { 'summary': { 'de': { 'errors': 0, 'warnings': 0, 'missing': 15, 'missing_w': 0, 'report': 0, 'obsolete': 0, 'changed': 0, 'changed_w': 0, 'unchanged': 0, 'unchanged_w': 0, 'keys': 0, } }, 'details': { 'de/sub/path': [{'missingEntity': 'one'}, {'missingEntity': 'two'}] } })
def test_module(self): obs = compare.Observer() f = paths.File('/some/real/sub/path', 'path', module='sub', locale='de') obs.notify('missingEntity', f, 'one') obs.notify('obsoleteEntity', f, 'bar') obs.notify('missingEntity', f, 'two') obs.updateStats(f, {'missing': 15}) self.assertDictEqual( obs.toJSON(), { 'summary': { 'de': { 'missing': 15 } }, 'details': { 'de/sub/path': [ { 'missingEntity': 'one' }, { 'obsoleteEntity': 'bar' }, { 'missingEntity': 'two' }, ] } })
def test_module(self): obs = compare.Observer(file_stats=True) f = paths.File('/some/real/sub/path', 'path', module='sub', locale='de') obs.notify('missingEntity', f, 'one') obs.notify('obsoleteEntity', f, 'bar') obs.notify('missingEntity', f, 'two') obs.updateStats(f, {'missing': 15}) self.assertDictEqual( obs.toJSON(), { 'summary': { 'de': { 'missing': 15 } }, 'details': { 'de/sub/path': [ { 'missingEntity': 'one' }, { 'obsoleteEntity': 'bar' }, { 'missingEntity': 'two' }, ] } }) self.assertDictEqual(obs.file_stats, {'de': { 'sub/path': { 'missing': 15 } }}) self.assertEqual( obs.serialize(), '''\ de/sub/path +one -bar +two de: missing: 15 0% of entries changed''') clone = loads(dumps(obs)) self.assertDictEqual(clone.summary, obs.summary) self.assertDictEqual(clone.details.toJSON(), obs.details.toJSON()) self.assertDictEqual(clone.file_stats, obs.file_stats)
def gather(self): fls = files.ProjectFiles("en-US", self.configs) results = [{} for cfg in self.configs] for l10n_path, f, _, tests in fls: if "android-dtd" in tests: # ignore Android native strings continue try: p = parser.getParser(f) except UserWarning: continue p.readFile(f) string_count = len(list(p)) for i, cfg in enumerate(self.configs): l10n_file = paths.File(l10n_path, mozpath.relpath(l10n_path, mozpath.abspath(".")), locale='en-US') if cfg.filter(l10n_file) != "ignore": results[i][l10n_file.file] = string_count return results
def compareProjects( project_configs, l10n_base_dir, stat_observer=None, merge_stage=None, clobber_merge=False, quiet=0, ): locales = set() comparer = ContentComparer(quiet) observers = comparer.observers for project in project_configs: # disable filter if we're in validation mode if None in project.locales: filter = None else: filter = project.filter observers.append(Observer( quiet=quiet, filter=filter, )) locales.update(project.locales) for locale in sorted(locales): files = paths.ProjectFiles(locale, project_configs, mergebase=merge_stage) if merge_stage is not None: if clobber_merge: mergematchers = set(_m.get('merge') for _m in files.matchers) mergematchers.discard(None) for matcher in mergematchers: clobberdir = matcher.prefix if os.path.exists(clobberdir): shutil.rmtree(clobberdir) print("clobbered " + clobberdir) for l10npath, refpath, mergepath, extra_tests in files: # module and file path are needed for legacy filter.py support module = None fpath = mozpath.relpath(l10npath, l10n_base_dir) for _m in files.matchers: if _m['l10n'].match(l10npath): if _m['module']: # legacy ini support, set module, and resolve # local path against the matcher prefix, # which includes the module module = _m['module'] fpath = mozpath.relpath(l10npath, _m['l10n'].prefix) break reffile = paths.File(refpath, fpath or refpath, module=module) if locale is None: # When validating the reference files, set locale # to a private subtag. This only shows in the output. locale = paths.REFERENCE_LOCALE l10n = paths.File(l10npath, fpath or l10npath, module=module, locale=locale) if not os.path.exists(l10npath): comparer.add(reffile, l10n, mergepath) continue if not os.path.exists(refpath): comparer.remove(reffile, l10n, mergepath) continue comparer.compare(reffile, l10n, mergepath, extra_tests) return observers