def test_references_wildcard(self, is_dir, glob_mock): parser = TOMLParser() def mock_load(ctx): ctx.data = { "locales": [], "paths": [ { "reference": "some/**/file.ftl", "l10n": "**/{android_locale}/file.ftl", }, ], } parser.load = mock.MagicMock(side_effect=mock_load) pc = parser.parse("a/basedir/l10n.toml") is_dir.side_effect = lambda p: not p.endswith("fixed/file.ftl") glob_mock.return_value = [ pc.root + f for f in [ "/some/other/content", "/some/other/file.ftl", "/some/second/deep/file.ftl", ] ] self.assertListEqual( process.references(pc, "a/basedir"), ["l10n.toml", "some/other/file.ftl", "some/second/deep/file.ftl"])
def load_configs(lintconfig, root, l10n_base): """Load l10n configuration files specified in the linter configuration.""" configs = [] env = {"l10n_base": l10n_base} for toml in lintconfig["l10n_configs"]: cfg = TOMLParser().parse( mozpath.join(root, toml), env=env, ignore_missing_includes=True ) cfg.set_locales([LOCALE], deep=True) configs.append(cfg) return configs
def test_references_file(self, is_dir, glob_mock): parser = TOMLParser() def mock_load(ctx): ctx.data = { "locales": [], "paths": [ { "reference": "some/fixed/file.ftl", "l10n": "{android_locale}/file.ftl", }, ], } parser.load = mock.MagicMock(side_effect=mock_load) pc = parser.parse("a/basedir/l10n.toml") is_dir.side_effect = lambda p: not p.endswith("fixed/file.ftl") glob_mock.side_effect = Exception("not called") self.assertListEqual(process.references(pc, "a/basedir"), ["l10n.toml", "some/fixed/file.ftl"])
def parsed_configuration(self): """Return parsed project configuration file.""" db_project = self.vcs_project.db_project path = os.path.join( db_project.source_repository.checkout_path, self.configuration_file, ) l10n_base = db_project.translation_repositories()[0].checkout_path return TOMLParser().parse(path, env={'l10n_base': l10n_base})
def handle(self, config_paths, l10n_base_dir, locales, merge=None, defines=None, unified=False, full=False, quiet=0, validate=False, clobber=False, data='text'): # using nargs multiple times in argparser totally screws things # up, repair that. # First files are configs, then the base dir, everything else is # locales all_args = config_paths + [l10n_base_dir] + locales config_paths = [] locales = [] if defines is None: defines = [] while all_args and not os.path.isdir(all_args[0]): config_paths.append(all_args.pop(0)) if not config_paths: self.parser.error('no configuration file given') for cf in config_paths: if not os.path.isfile(cf): self.parser.error('config file %s not found' % cf) if not all_args: self.parser.error('l10n-base-dir not found') l10n_base_dir = all_args.pop(0) if validate: # signal validation mode by setting locale list to [None] locales = [None] else: locales.extend(all_args) # when we compare disabled projects, we set our locales # on all subconfigs, so deep is True. locales_deep = full configs = [] config_env = {} for define in defines: var, _, value = define.partition('=') config_env[var] = value for config_path in config_paths: if config_path.endswith('.toml'): try: config = TOMLParser.parse(config_path, env=config_env) except ConfigNotFound as e: self.parser.exit('config file %s not found' % e.filename) config.add_global_environment(l10n_base=l10n_base_dir) if locales: config.set_locales(locales, deep=locales_deep) configs.append(config) else: app = EnumerateApp(config_path, l10n_base_dir, locales) configs.append(app.asConfig()) try: unified_observer = None if unified: unified_observer = Observer(quiet=quiet) observers = compareProjects(configs, quiet=quiet, stat_observer=unified_observer, merge_stage=merge, clobber_merge=clobber) except (OSError, IOError) as exc: print("FAIL: " + str(exc)) self.parser.exit(2) if unified: observers = [unified_observer] rv = 0 for observer in observers: print(observer.serialize(type=data)) # summary is a dict of lang-summary dicts # find out if any of our results has errors, return 1 if so if rv > 0: continue # we already have errors for loc, summary in observer.summary.items(): if summary.get('errors', 0) > 0: rv = 1 # no need to check further summaries, but # continue to run through observers break return rv
def copy_toml(self): shutil.copy2(self.src_toml, self.dest_toml) self.dest_config = TOMLParser().parse(self.dest_toml)
def read_configs(self): parser = TOMLParser() self.src_config = parser.parse(self.src_toml) self.dest_config = parser.parse(self.dest_toml)
def handle( self, quiet=0, verbose=0, validate=False, merge=None, config_paths=[], l10n_base_dir=None, locales=[], defines=[], full=False, return_zero=False, clobber=False, json=None, ): """The instance part of the classmethod call. Using keyword arguments as that is what we need for mach commands in mozilla-central. """ # log as verbose or quiet as we want, warn by default logging_level = logging.WARNING - (verbose - quiet) * 10 logging.basicConfig() logging.getLogger().setLevel(logging_level) config_paths, l10n_base_dir, locales = self.extract_positionals( validate=validate, config_paths=config_paths, l10n_base_dir=l10n_base_dir, locales=locales, ) # when we compare disabled projects, we set our locales # on all subconfigs, so deep is True. locales_deep = full configs = [] config_env = { 'l10n_base': l10n_base_dir } for define in defines: var, _, value = define.partition('=') config_env[var] = value for config_path in config_paths: if config_path.endswith('.toml'): try: config = TOMLParser().parse(config_path, env=config_env) except ConfigNotFound as e: self.parser.exit('config file %s not found' % e.filename) if locales_deep: if not locales: # no explicit locales given, force all locales config.set_locales(config.all_locales, deep=True) else: config.set_locales(locales, deep=True) configs.append(config) else: app = EnumerateApp(config_path, l10n_base_dir) configs.append(app.asConfig()) try: observers = compareProjects( configs, locales, l10n_base_dir, quiet=quiet, merge_stage=merge, clobber_merge=clobber) except (OSError, IOError) as exc: print("FAIL: " + str(exc)) self.parser.exit(2) if json is None or json != '-': details = observers.serializeDetails() if details: print(details) if len(configs) > 1: if details: print('') print("Summaries for") for config_path in config_paths: print(" " + config_path) print(" and the union of these, counting each string once") print(observers.serializeSummaries()) if json is not None: data = [observer.toJSON() for observer in observers] stdout = json == '-' indent = 1 if stdout else None fh = sys.stdout if stdout else open(json, 'w') json_dump(data, fh, sort_keys=True, indent=indent) if stdout: fh.write('\n') fh.close() rv = 1 if not return_zero and observers.error else 0 return rv
def parsed_configuration(self): """Return parsed project configuration file.""" return TOMLParser().parse( self.configuration_path, env={'l10n_base': self.l10n_base}, )
def handle( self, quiet=0, verbose=0, validate=False, merge=None, config_paths=[], l10n_base_dir=None, locales=[], defines=[], full=False, return_zero=False, clobber=False, json=None, ): """The instance part of the classmethod call. Using keyword arguments as that is what we need for mach commands in mozilla-central. """ # log as verbose or quiet as we want, warn by default logging_level = logging.WARNING - (verbose - quiet) * 10 logging.basicConfig() logging.getLogger().setLevel(logging_level) config_paths, l10n_base_dir, locales = self.extract_positionals( validate=validate, config_paths=config_paths, l10n_base_dir=l10n_base_dir, locales=locales, ) # when we compare disabled projects, we set our locales # on all subconfigs, so deep is True. locales_deep = full configs = [] config_env = {'l10n_base': l10n_base_dir} for define in defines: var, _, value = define.partition('=') config_env[var] = value for config_path in config_paths: if config_path.endswith('.toml'): try: config = TOMLParser().parse(config_path, env=config_env) except ConfigNotFound as e: self.parser.exit('config file %s not found' % e.filename) if locales_deep: if not locales: # no explicit locales given, force all locales config.set_locales(config.all_locales, deep=True) else: config.set_locales(locales, deep=True) configs.append(config) else: app = EnumerateApp(config_path, l10n_base_dir) configs.append(app.asConfig()) try: observers = compareProjects(configs, locales, l10n_base_dir, quiet=quiet, merge_stage=merge, clobber_merge=clobber) except (OSError, IOError) as exc: print("FAIL: " + str(exc)) self.parser.exit(2) if json is None or json != '-': details = observers.serializeDetails() if details: print(details) if len(configs) > 1: if details: print('') print("Summaries for") for config_path in config_paths: print(" " + config_path) print(" and the union of these, counting each string once") print(observers.serializeSummaries()) if json is not None: data = [observer.toJSON() for observer in observers] stdout = json == '-' indent = 1 if stdout else None fh = sys.stdout if stdout else open(json, 'w') json_dump(data, fh, sort_keys=True, indent=indent) if stdout: fh.write('\n') fh.close() rv = 1 if not return_zero and observers.error else 0 return rv
def gather_repo(self, repo): basepath = repo.path pc = TOMLParser().parse(mozpath.join(basepath, "l10n.toml")) paths = ["l10n.toml"] + [ mozpath.relpath( m["reference"].pattern.expand(m["reference"].env), basepath ) for m in pc.paths ] self.paths_for_repos[repo.name] = paths branches = repo.branches() self.branches[repo.name] = branches[:] known_revs = self.revs.get(repo.name, {}) for branch_num in range(len(branches)): branch = branches[branch_num] prior_branches = branches[:branch_num] cmd = [ "git", "-C", basepath, "log", "--parents", "--format=%H %ct %P" ] + [ "^" + repo.ref(b) for b in prior_branches ] if branch in known_revs: cmd += ["^" + known_revs[branch]] block_revs = [] elif branch_num == 0: # We haven't seen this repo yet. # Block all known revs in the target from being converted again # in case of repository-level forks. block_revs = self.target.known_revs() cmd += [repo.ref(branch), "--"] + paths out = subprocess.run( cmd, stdout=subprocess.PIPE, encoding="ascii" ).stdout for commit_line in out.splitlines(): segs = commit_line.split() commit = segs.pop(0) if commit in block_revs: continue commit_date = int(segs.pop(0)) self.repos_for_hash[commit].append((repo.name, branch)) self.hashes_for_repo[repo.name].add(commit) self.commit_dates[commit] = max( commit_date, self.commit_dates.get(commit, 0) ) for parent in segs: self.parents[commit].add(parent) self.children[parent].add(commit) if branch in known_revs or branch_num == 0: continue # We don't know this branch yet, and it's a fork. # Find the branch point to the previous branches. for prior_branch in prior_branches: cmd = [ "git", "-C", basepath, "merge-base", repo.ref(branch), repo.ref(prior_branch), ] branch_rev = subprocess.run( cmd, stdout=subprocess.PIPE, encoding="ascii" ).stdout.strip() if not branch_rev: continue # We have a branch revision, find the next child on the # route to the prior branch to add that to. cmd = [ "git", "-C", basepath, "rev-list", "-n", "1", "{}..{}".format(branch_rev, repo.ref(prior_branch)), ] fork_rev = subprocess.run( cmd, stdout=subprocess.PIPE, encoding="ascii" ).stdout.strip() if fork_rev: self.forks[fork_rev].append( (repo.name, branch, branch_rev) )
def handle(self, config_paths, l10n_base_dir, locales, merge=None, defines=None, unified=False, full=False, quiet=0, clobber=False, data='text'): # using nargs multiple times in argparser totally screws things # up, repair that. # First files are configs, then the base dir, everything else is # locales all_args = config_paths + [l10n_base_dir] + locales config_paths = [] locales = [] if defines is None: defines = [] while all_args and not os.path.isdir(all_args[0]): config_paths.append(all_args.pop(0)) if not config_paths: self.parser.error('no configuration file given') for cf in config_paths: if not os.path.isfile(cf): self.parser.error('config file %s not found' % cf) if not all_args: self.parser.error('l10n-base-dir not found') l10n_base_dir = all_args.pop(0) locales.extend(all_args) # when we compare disabled projects, we set our locales # on all subconfigs, so deep is True. locales_deep = full configs = [] config_env = {} for define in defines: var, _, value = define.partition('=') config_env[var] = value for config_path in config_paths: if config_path.endswith('.toml'): try: config = TOMLParser.parse(config_path, env=config_env) except ConfigNotFound as e: self.parser.exit('config file %s not found' % e.filename) config.add_global_environment(l10n_base=l10n_base_dir) if locales: config.set_locales(locales, deep=locales_deep) configs.append(config) else: app = EnumerateApp(config_path, l10n_base_dir, locales) configs.append(app.asConfig()) try: unified_observer = None if unified: unified_observer = Observer(quiet=quiet) observers = compareProjects(configs, quiet=quiet, stat_observer=unified_observer, merge_stage=merge, clobber_merge=clobber) except (OSError, IOError), exc: print "FAIL: " + str(exc) self.parser.exit(2)
def get_config(toml_path): c = TOMLParser().parse(toml_path) return c
def compare(browser_toml, l10n_base_dir): config_env = {"l10n_base": l10n_base_dir} configs = [TOMLParser().parse(browser_toml, env=config_env)] return compareProjects(configs, ["ja"], l10n_base_dir).toJSON()