def test_load_json_or_yaml(string, is_path, exception, raises, result): """Exercise ``load_json_or_yaml`` various options.""" if raises: with pytest.raises(exception): utils.load_json_or_yaml(string, is_path=is_path, exception=exception) else: for file_type in ("json", "yaml"): assert result == utils.load_json_or_yaml(string, is_path=is_path, exception=exception, file_type=file_type)
def verify_task_schema(config, task, schema_key="schema_file"): """Verify the task definition. Args: config (dict): the running config task (dict): the running task schema_key: the key in `config` where the path to the schema file is. Key can contain dots (e.g.: 'schema_files.file_a') Raises: TaskVerificationError: if the task doesn't match the schema """ schema_path = config schema_keys = schema_key.split(".") try: for key in schema_keys: schema_path = schema_path[key] task_schema = load_json_or_yaml(schema_path, is_path=True) log.debug("Task is verified against this schema: {}".format(task_schema)) verify_json_schema(task, task_schema) except (KeyError, OSError) as e: raise TaskVerificationError( "Cannot verify task against schema. Task: {}.".format(task) ) from e
def init_config(config_path=None, default_config=None, validator_callback=None): """Initialize the config. First, read config overrides from ``config_path``. Apply over ``default_config``. Send to ``validator_config``, then return a immutabledict of the config. Args: config_path (str, optional): the path to the config file. Defaults to ``sys.argv[1]``. default_config (dict, optional): the config defaults. These are the config values if not overridden in ``config_path``. Defaults to ``{}``. validator_callback (function, optional): a function that takes a single arg (``config``), and raises an exception if invalid. If ``None``, don't validate the config. Defaults to ``None``. Raises: Exception: if the config doesn't pass the ``validator_callback``. Returns: immutabledict: the config. """ if config_path is None: if len(sys.argv) != 2: _usage() config_path = sys.argv[1] config = {} if default_config is None else dict(default_config) config.update(load_json_or_yaml(config_path, file_type="yaml", is_path=True)) validator_callback and validator_callback(config) return immutabledict(config)
async def check_treestatus(config, task): """Return True if we can land based on treestatus. Args: config (dict): the running config task (dict): the running task Returns: bool: ``True`` if the tree is open. """ tree = get_short_source_repo(task) url = "%s/trees/%s" % (config["treestatus_base_url"], tree) path = os.path.join(config["work_dir"], "treestatus.json") await retry_async(download_file, args=(url, path), retry_exceptions=(DownloadError, )) treestatus = load_json_or_yaml(path, is_path=True) if treestatus["result"]["status"] != "closed": log.info( "treestatus is %s - assuming we can land", repr(treestatus["result"]["status"]), ) return True return False
async def l10n_bump(config, task, repo_path): """Perform a version bump. This function takes its inputs from task by using the ``get_l10n_bump_info`` function from treescript.task. Using `next_version` and `files`. This function does nothing (but logs) if the current version and next version match, and nothing if the next_version is actually less than current_version. Args: config (dict): the running config task (dict): the running task repo_path (str): the source directory Raises: TaskVerificationError: if a file specified is not allowed, or if the file is not in the target repository. Returns: bool: True if there are any changes. """ log.info("Preparing to bump l10n changesets.") dontbuild = get_dontbuild(task) ignore_closed_tree = get_ignore_closed_tree(task) l10n_bump_info = get_l10n_bump_info(task) revision_info = None changes = 0 if not ignore_closed_tree: if not await check_treestatus(config, task): log.info("Treestatus is closed; skipping l10n bump.") return for bump_config in l10n_bump_info: if bump_config.get("revision_url"): revision_info = await get_revision_info(bump_config, repo_path) path = os.path.join(repo_path, bump_config["path"]) old_contents = load_json_or_yaml(path, is_path=True) new_contents = build_revision_dict(bump_config, revision_info, repo_path) if old_contents == new_contents: continue with open(path, "w") as fh: fh.write( json.dumps(new_contents, sort_keys=True, indent=4, separators=(",", ": "))) locale_map = build_locale_map(old_contents, new_contents) message = build_commit_message(bump_config["name"], locale_map, dontbuild=dontbuild, ignore_closed_tree=ignore_closed_tree) await run_hg_command(config, "commit", "-m", message, repo_path=repo_path) changes += 1 return changes
async def l10n_bump(config, task, repo_path, repo_type="hg"): """Perform a l10n revision bump. This function takes its inputs from task by using the ``get_l10n_bump_info`` function from treescript.task. It then calculates the locales, the platforms for each locale, and the locale revision for each locale. Args: config (dict): the running config task (dict): the running task repo_path (str): the source directory repo_type (str): the repository type Raises: TaskVerificationError: if a file specified is not allowed, or if the file is not in the target repository. Returns: int: non-zero if there are any changes. """ vcs = get_vcs_module(repo_type) log.info("Preparing to bump l10n changesets.") ignore_closed_tree = get_ignore_closed_tree(task) if not ignore_closed_tree: if not await check_treestatus(config, task): log.info("Treestatus is closed; skipping l10n bump.") return 0 dontbuild = get_dontbuild(task) l10n_bump_info = get_l10n_bump_info(task) changes = 0 for bump_config in l10n_bump_info: path = os.path.join(repo_path, bump_config["path"]) old_contents = load_json_or_yaml(path, is_path=True) new_contents = await build_revision_dict(bump_config, repo_path, deepcopy(old_contents)) if old_contents == new_contents: continue with open(path, "w") as fh: fh.write( json.dumps(new_contents, sort_keys=True, indent=4, separators=(",", ": "))) locale_map = build_locale_map(old_contents, new_contents) message = build_commit_message(bump_config["name"], locale_map, dontbuild=dontbuild, ignore_closed_tree=ignore_closed_tree) await vcs.commit(config, repo_path, message) changes += 1 return changes
def _init_config(config_path=None, default_config=None): if config_path is None: if len(sys.argv) != 2: _usage() config_path = sys.argv[1] config = {} if default_config is None else default_config config.update( load_json_or_yaml(config_path, file_type="yaml", is_path=True)) return config
async def download_uuids(self): """Download the UUID manifest.""" payload = self.claim_task["task"]["payload"] if payload.get("uuids"): # enable specifying uuids directly, for integration tests uuids = payload["uuids"] else: url = self.claim_task["task"]["payload"]["uuid_manifest"] path = os.path.join(self.task_dir, "uuids.json") self.task_log("Downloading %s", url) await retry_async(download_file, args=(url, path), retry_exceptions=(DownloadError,)) uuids = load_json_or_yaml(path, is_path=True) self.uuids = tuple(uuids) self.task_log("UUIDs: %s", self.uuids)
async def get_existing_tags(config, repo_path): """Get the existing tags in a mercurial repo. Args: config (dict): the running config repo_path (str): the path to the repo Returns: dict: ``{tag1: revision1, tag2: revision2, ...}`` """ existing_tags = {} output = load_json_or_yaml(await run_hg_command(config, "tags", "--template=json", repo_path=repo_path, return_output=True)) for tag_info in output: existing_tags[tag_info["tag"]] = tag_info["node"] return existing_tags
def get_task(config): """Read the task.json from work_dir. Args: config (dict): the running config, to find work_dir. Returns: dict: the contents of task.json Raises: ClientError: on error. """ path = os.path.join(config["work_dir"], "task.json") message = "Can't read task from {}!\n%(exc)s".format(path) contents = load_json_or_yaml(path, is_path=True, message=message) return contents
async def get_latest_revision(locale, url): """Download the hg pushlog for the latest locale revision. Args: locale (str): the locale to query url (str): the [templatized] pushlog url Returns: tuple (locale, revision) """ url = url % {"locale": locale} with tempfile.NamedTemporaryFile() as fp: path = fp.name await retry_async(download_file, args=(url, path), retry_exceptions=(DownloadError, )) revision_info = load_json_or_yaml(path, is_path=True) last_push_id = revision_info["lastpushid"] revision = revision_info["pushes"][str(last_push_id)]["changesets"][0] log.info(f"locale {locale} revision {revision}") return (locale, revision)