def test_load_json_or_yaml(string, is_path, exception, raises, result):
"""Exercise ``load_json_or_yaml`` various options."""
if raises:
with pytest.raises(exception):
utils.load_json_or_yaml(string,
is_path=is_path,
exception=exception)
else:
for file_type in ("json", "yaml"):
assert result == utils.load_json_or_yaml(string,
is_path=is_path,
exception=exception,
file_type=file_type)
def verify_task_schema(config, task, schema_key="schema_file"):
"""Verify the task definition.
Args:
config (dict): the running config
task (dict): the running task
schema_key: the key in `config` where the path to the schema file is. Key can contain
dots (e.g.: 'schema_files.file_a')
Raises:
TaskVerificationError: if the task doesn't match the schema
"""
schema_path = config
schema_keys = schema_key.split(".")
try:
for key in schema_keys:
schema_path = schema_path[key]
task_schema = load_json_or_yaml(schema_path, is_path=True)
log.debug("Task is verified against this schema: {}".format(task_schema))
verify_json_schema(task, task_schema)
except (KeyError, OSError) as e:
raise TaskVerificationError(
"Cannot verify task against schema. Task: {}.".format(task)
) from e
def init_config(config_path=None, default_config=None, validator_callback=None):
"""Initialize the config.
First, read config overrides from ``config_path``. Apply over
``default_config``. Send to ``validator_config``, then return a immutabledict
of the config.
Args:
config_path (str, optional): the path to the config file. Defaults to
``sys.argv[1]``.
default_config (dict, optional): the config defaults. These are the
config values if not overridden in ``config_path``. Defaults to
``{}``.
validator_callback (function, optional): a function that takes a single
arg (``config``), and raises an exception if invalid. If ``None``,
don't validate the config. Defaults to ``None``.
Raises:
Exception: if the config doesn't pass the ``validator_callback``.
Returns:
immutabledict: the config.
"""
if config_path is None:
if len(sys.argv) != 2:
_usage()
config_path = sys.argv[1]
config = {} if default_config is None else dict(default_config)
config.update(load_json_or_yaml(config_path, file_type="yaml", is_path=True))
validator_callback and validator_callback(config)
return immutabledict(config)
async def check_treestatus(config, task):
"""Return True if we can land based on treestatus.
Args:
config (dict): the running config
task (dict): the running task
Returns:
bool: ``True`` if the tree is open.
"""
tree = get_short_source_repo(task)
url = "%s/trees/%s" % (config["treestatus_base_url"], tree)
path = os.path.join(config["work_dir"], "treestatus.json")
await retry_async(download_file,
args=(url, path),
retry_exceptions=(DownloadError, ))
treestatus = load_json_or_yaml(path, is_path=True)
if treestatus["result"]["status"] != "closed":
log.info(
"treestatus is %s - assuming we can land",
repr(treestatus["result"]["status"]),
)
return True
return False
async def l10n_bump(config, task, repo_path):
"""Perform a version bump.
This function takes its inputs from task by using the ``get_l10n_bump_info``
function from treescript.task. Using `next_version` and `files`.
This function does nothing (but logs) if the current version and next version
match, and nothing if the next_version is actually less than current_version.
Args:
config (dict): the running config
task (dict): the running task
repo_path (str): the source directory
Raises:
TaskVerificationError: if a file specified is not allowed, or
if the file is not in the target repository.
Returns:
bool: True if there are any changes.
"""
log.info("Preparing to bump l10n changesets.")
dontbuild = get_dontbuild(task)
ignore_closed_tree = get_ignore_closed_tree(task)
l10n_bump_info = get_l10n_bump_info(task)
revision_info = None
changes = 0
if not ignore_closed_tree:
if not await check_treestatus(config, task):
log.info("Treestatus is closed; skipping l10n bump.")
return
for bump_config in l10n_bump_info:
if bump_config.get("revision_url"):
revision_info = await get_revision_info(bump_config, repo_path)
path = os.path.join(repo_path, bump_config["path"])
old_contents = load_json_or_yaml(path, is_path=True)
new_contents = build_revision_dict(bump_config, revision_info,
repo_path)
if old_contents == new_contents:
continue
with open(path, "w") as fh:
fh.write(
json.dumps(new_contents,
sort_keys=True,
indent=4,
separators=(",", ": ")))
locale_map = build_locale_map(old_contents, new_contents)
message = build_commit_message(bump_config["name"],
locale_map,
dontbuild=dontbuild,
ignore_closed_tree=ignore_closed_tree)
await run_hg_command(config,
"commit",
"-m",
message,
repo_path=repo_path)
changes += 1
return changes
async def l10n_bump(config, task, repo_path, repo_type="hg"):
"""Perform a l10n revision bump.
This function takes its inputs from task by using the ``get_l10n_bump_info``
function from treescript.task. It then calculates the locales, the platforms
for each locale, and the locale revision for each locale.
Args:
config (dict): the running config
task (dict): the running task
repo_path (str): the source directory
repo_type (str): the repository type
Raises:
TaskVerificationError: if a file specified is not allowed, or
if the file is not in the target repository.
Returns:
int: non-zero if there are any changes.
"""
vcs = get_vcs_module(repo_type)
log.info("Preparing to bump l10n changesets.")
ignore_closed_tree = get_ignore_closed_tree(task)
if not ignore_closed_tree:
if not await check_treestatus(config, task):
log.info("Treestatus is closed; skipping l10n bump.")
return 0
dontbuild = get_dontbuild(task)
l10n_bump_info = get_l10n_bump_info(task)
changes = 0
for bump_config in l10n_bump_info:
path = os.path.join(repo_path, bump_config["path"])
old_contents = load_json_or_yaml(path, is_path=True)
new_contents = await build_revision_dict(bump_config, repo_path,
deepcopy(old_contents))
if old_contents == new_contents:
continue
with open(path, "w") as fh:
fh.write(
json.dumps(new_contents,
sort_keys=True,
indent=4,
separators=(",", ": ")))
locale_map = build_locale_map(old_contents, new_contents)
message = build_commit_message(bump_config["name"],
locale_map,
dontbuild=dontbuild,
ignore_closed_tree=ignore_closed_tree)
await vcs.commit(config, repo_path, message)
changes += 1
return changes
def _init_config(config_path=None, default_config=None):
if config_path is None:
if len(sys.argv) != 2:
_usage()
config_path = sys.argv[1]
config = {} if default_config is None else default_config
config.update(
load_json_or_yaml(config_path, file_type="yaml", is_path=True))
return config
async def download_uuids(self):
"""Download the UUID manifest."""
payload = self.claim_task["task"]["payload"]
if payload.get("uuids"):
# enable specifying uuids directly, for integration tests
uuids = payload["uuids"]
else:
url = self.claim_task["task"]["payload"]["uuid_manifest"]
path = os.path.join(self.task_dir, "uuids.json")
self.task_log("Downloading %s", url)
await retry_async(download_file, args=(url, path), retry_exceptions=(DownloadError,))
uuids = load_json_or_yaml(path, is_path=True)
self.uuids = tuple(uuids)
self.task_log("UUIDs: %s", self.uuids)
async def get_existing_tags(config, repo_path):
"""Get the existing tags in a mercurial repo.
Args:
config (dict): the running config
repo_path (str): the path to the repo
Returns:
dict: ``{tag1: revision1, tag2: revision2, ...}``
"""
existing_tags = {}
output = load_json_or_yaml(await run_hg_command(config, "tags", "--template=json", repo_path=repo_path, return_output=True))
for tag_info in output:
existing_tags[tag_info["tag"]] = tag_info["node"]
return existing_tags
def get_task(config):
"""Read the task.json from work_dir.
Args:
config (dict): the running config, to find work_dir.
Returns:
dict: the contents of task.json
Raises:
ClientError: on error.
"""
path = os.path.join(config["work_dir"], "task.json")
message = "Can't read task from {}!\n%(exc)s".format(path)
contents = load_json_or_yaml(path, is_path=True, message=message)
return contents
async def get_latest_revision(locale, url):
"""Download the hg pushlog for the latest locale revision.
Args:
locale (str): the locale to query
url (str): the [templatized] pushlog url
Returns:
tuple (locale, revision)
"""
url = url % {"locale": locale}
with tempfile.NamedTemporaryFile() as fp:
path = fp.name
await retry_async(download_file,
args=(url, path),
retry_exceptions=(DownloadError, ))
revision_info = load_json_or_yaml(path, is_path=True)
last_push_id = revision_info["lastpushid"]
revision = revision_info["pushes"][str(last_push_id)]["changesets"][0]
log.info(f"locale {locale} revision {revision}")
return (locale, revision)
