def test_get_source_repo_no_source(task_defn):
del task_defn['metadata']['source']
with pytest.raises(TaskVerificationError):
stask.get_source_repo(task_defn)
del task_defn['metadata']
with pytest.raises(TaskVerificationError):
stask.get_source_repo(task_defn)
def test_get_source_repo_no_source(task_defn):
del task_defn["metadata"]["source"]
with pytest.raises(TaskVerificationError):
ttask.get_source_repo(task_defn)
del task_defn["metadata"]
with pytest.raises(TaskVerificationError):
ttask.get_source_repo(task_defn)
async def checkout_repo(context, directory):
"""Perform a clone via robustcheckout, at ${directory}/src.
This function will perform a clone via robustcheckout, using hg's share extension
for a cache at 'context.config['hg_share_base_dir']' that robustcheckout will
populate if necessary.
Robustcheckout will retry network operations at most 3 times (robustcheckout's default)
before giving up and causing FailedSubprocess to be raised.
Args:
context (TreeScriptContext): the treescript context
directory (str): The directory to place the resulting clone.
Raises:
FailedSubprocess: if the clone attempt doesn't succeed.
"""
share_base = context.config['hg_share_base_dir']
upstream_repo = context.config['upstream_repo']
dest_repo = get_source_repo(context.task)
dest_folder = os.path.join(directory, 'src')
context.repo = dest_folder
# branch default is used to pull tip of the repo at checkout time
await run_hg_command(context, 'robustcheckout', dest_repo, dest_folder,
'--sharebase', share_base,
'--upstream', upstream_repo,
'--branch', 'default')
async def checkout_repo(config, task, repo_path):
"""Perform a clone via robustcheckout, at ${directory}/src.
This function will perform a clone via robustcheckout, using hg's share extension
for a cache at 'config['hg_share_base_dir']' that robustcheckout will
populate if necessary.
Robustcheckout will retry network operations at most 3 times (robustcheckout's default)
before giving up and causing FailedSubprocess to be raised.
Args:
config (dict): the running config.
task (dict): the running task.
repo_path (str): The directory to place the resulting clone.
Raises:
CheckoutError: if the clone attempt doesn't succeed.
"""
share_base = config["hg_share_base_dir"]
upstream_repo = config["upstream_repo"]
source_repo = get_source_repo(task)
# branch default is used to pull tip of the repo at checkout time
branch = get_branch(task, "default")
await run_hg_command(config,
"robustcheckout",
source_repo,
repo_path,
"--sharebase",
share_base,
"--upstream",
upstream_repo,
"--branch",
branch,
exception=CheckoutError)
async def checkout_repo(context, directory):
"""Perform a clone via robustcheckout, at ${directory}/src.
This function will perform a clone via robustcheckout, using hg's share extension
for a cache at 'context.config['hg_share_base_dir']' that robustcheckout will
populate if necessary.
Robustcheckout will retry network operations at most 3 times (robustcheckout's default)
before giving up and causing FailedSubprocess to be raised.
Args:
context (TreeScriptContext): the treescript context
directory (str): The directory to place the resulting clone.
Raises:
FailedSubprocess: if the clone attempt doesn't succeed.
"""
share_base = context.config['hg_share_base_dir']
upstream_repo = context.config['upstream_repo']
dest_repo = get_source_repo(context.task)
dest_folder = os.path.join(directory, 'src')
context.repo = dest_folder
# branch default is used to pull tip of the repo at checkout time
await run_hg_command(context, 'robustcheckout', dest_repo, dest_folder,
'--sharebase', share_base, '--upstream',
upstream_repo, '--branch', 'default')
async def log_outgoing(config, task, repo_path):
"""Run `hg out` against the current revision in the repository.
This logs current changes that will be pushed (or would have been, if dry-run)
Args:
config (dict): the running config
task (dict): the running task
repo_path (str): the source repo path
Raises:
FailedSubprocess: on failure
"""
dest_repo = get_source_repo(task)
log.info("outgoing changesets..")
output = await run_hg_command(
config,
"out",
"-vp",
"-r",
".",
dest_repo,
repo_path=repo_path,
return_output=True,
)
if output:
path = os.path.join(config["artifact_dir"], "public", "logs", "outgoing.diff")
makedirs(os.path.dirname(path))
with open(path, "w") as fh:
fh.write(output)
async def do_tagging(config, task, repo_path):
"""Perform tagging, at ${repo_path}/src.
This function will perform a mercurial tag, on 'default' head of target repository.
It will tag the revision specified in the tag_info portion of the task payload, using
the specified tags in that payload.
Tags are forced to be created at the specified revision if they already existed.
This function has the side affect of pulling the specified revision from the
destination repository. This feature exists because mozilla-unified does not
contain relbranches, though some releases are created on relbranches, so we must ensure
the desired revision to tag is known to the local repository.
Args:
config (dict): the running config.
task (dict): the running task.
repo_path (str): The directory to place the resulting clone.
Raises:
FailedSubprocess: if the tag attempt doesn't succeed.
Returns:
bool: True if there are any changes.
"""
tag_info = get_tag_info(task)
desired_tags = await check_tags(config, tag_info, repo_path)
if not desired_tags:
log.info("No unique tags to add; skipping tagging.")
return
desired_rev = tag_info["revision"]
dontbuild = get_dontbuild(task)
dest_repo = get_source_repo(task)
commit_msg = TAG_MSG.format(revision=desired_rev, tags=", ".join(desired_tags))
if dontbuild:
commit_msg += DONTBUILD_MSG
log.info(
"Pulling {revision} from {repo} explicitly.".format(
revision=desired_rev, repo=dest_repo
)
)
await run_hg_command(
config, "pull", "-r", desired_rev, dest_repo, repo_path=repo_path
)
log.info(commit_msg)
await run_hg_command(
config,
"tag",
"-m",
commit_msg,
"-r",
desired_rev,
"-f", # Todo only force if needed
*desired_tags,
repo_path=repo_path,
)
return True
async def log_outgoing(context, directory):
"""Run `hg out` against the current revision in the repository.
This logs current changes that will be pushed (or would have been, if dry-run)
"""
local_repo = os.path.join(directory, 'src')
dest_repo = get_source_repo(context.task)
log.info("outgoing changesets..")
await run_hg_command(context, 'out', '-vp', '-r', '.', dest_repo, local_repo=local_repo)
async def log_outgoing(context, directory):
"""Run `hg out` against the current revision in the repository.
This logs current changes that will be pushed (or would have been, if dry-run)
"""
local_repo = os.path.join(directory, 'src')
dest_repo = get_source_repo(context.task)
log.info("outgoing changesets..")
await run_hg_command(context, 'out', '-vp', '-r', '.', dest_repo, local_repo=local_repo)
async def do_actions(config, task, actions, repo_path):
"""Perform the set of actions that treescript can perform.
The actions happen in order, tagging, ver bump, then push
Args:
config (dict): the running config
task (dict): the running task
actions (list): the actions to perform
repo_path (str): the source directory to use.
"""
source_repo = get_source_repo(task)
# mercurial had been the only default choice until git was supported, default to it.
repo_type = "git" if is_github_url(source_repo) else "hg"
vcs = get_vcs_module(repo_type)
await vcs.checkout_repo(config, task, repo_path)
# Split the action selection up due to complexity in do_actions
# caused by different push behaviour, and action return values.
if "merge_day" in actions:
await perform_merge_actions(config, task, actions, repo_path, repo_type)
return
num_changes = 0
if "tag" in actions:
num_changes += await vcs.do_tagging(config, task, repo_path)
if "version_bump" in actions:
num_changes += await bump_version(config, task, repo_path, repo_type)
if "l10n_bump" in actions:
num_changes += await l10n_bump(config, task, repo_path, repo_type)
num_outgoing = await vcs.log_outgoing(config, task, repo_path)
if num_outgoing != num_changes:
raise TreeScriptError("Outgoing changesets don't match number of expected changesets!" " {} vs {}".format(num_outgoing, num_changes))
if should_push(task, actions):
if num_changes:
await vcs.push(config, task, repo_path, target_repo=get_source_repo(task))
else:
log.info("No changes; skipping push.")
await vcs.strip_outgoing(config, task, repo_path)
async def checkout_repo(config, task, repo_path):
"""Perform a git clone at ${directory}/src.
This function will perform a git clone. It will also checkout to the right branch,
if provided in the task definition.
Args:
config (dict): the running config.
task (dict): the running task.
repo_path (str): The directory to place the resulting clone.
Raises:
TaskVerificationError: if the branch does not exist upstream.
"""
source_repo = get_source_repo(task)
if os.path.exists(repo_path):
log.info("Reusing existing repo_path: {}".format(repo_path))
repo = Repo(repo_path)
else:
log.info('Cloning source_repo "{}" to repo_path: {}'.format(
source_repo, repo_path))
repo = Repo.clone_from(
source_repo,
repo_path,
)
branch = get_branch(task)
if branch:
# GitPython cannot simply `git checkout` to right upstream branch. We have to manually
# create a new branch and manually set the upstream branch
log.info('Checking out branch "{}"'.format(branch))
remote_branches = repo.remotes.origin.fetch()
remote_branches_names = [
fetch_info.name for fetch_info in remote_branches
]
remote_branch = get_single_item_from_sequence(
remote_branches_names,
condition=lambda remote_branch_name: remote_branch_name ==
_get_upstream_branch_name(branch),
ErrorClass=TaskVerificationError,
no_item_error_message="Branch does not exist on remote repo",
too_many_item_error_message="Too many branches with that name",
)
repo.create_head(branch, remote_branch)
repo.branches[branch].checkout()
else:
log.warn(
"No branch provided in the task payload. Staying on the default one"
)
async def do_tagging(context, directory):
"""Perform tagging, at ${directory}/src.
This function will perform a mercurial tag, on 'default' head of target repository.
It will tag the revision specified in the tag_info portion of the task payload, using
the specified tags in that payload.
Tags are forced to be created at the specified revision if they already existed.
This function has the side affect of pulling the specified revision from the
destination repository. This feature exists because mozilla-unified does not
contain relbranches, though some releases are created on relbranches, so we must ensure
the desired revision to tag is known to the local repository.
Args:
context (TreeScriptContext): the treescript context
directory (str): The directory to place the resulting clone.
Raises:
FailedSubprocess: if the tag attempt doesn't succeed.
"""
local_repo = os.path.join(directory, 'src')
tag_info = get_tag_info(context.task)
desired_tags = tag_info['tags']
desired_rev = tag_info['revision']
dontbuild = get_dontbuild(context.task)
dest_repo = get_source_repo(context.task)
commit_msg = TAG_MSG.format(revision=desired_rev,
tags=', '.join(desired_tags))
if dontbuild:
commit_msg += DONTBUILD_MSG
log.info("Pulling {revision} from {repo} explicitly.".format(
revision=desired_rev, repo=dest_repo))
await run_hg_command(context,
'pull',
'-r',
desired_rev,
dest_repo,
local_repo=local_repo)
log.info(commit_msg)
await run_hg_command(
context,
'tag',
'-m',
commit_msg,
'-r',
desired_rev,
'-f', # Todo only force if needed
*desired_tags,
local_repo=local_repo)
async def push(context):
"""Run `hg push` against the current source repo."""
local_repo = context.repo
dest_repo = get_source_repo(context.task)
dest_repo_ssh = dest_repo.replace('https://', 'ssh://')
ssh_username = context.config.get("hg_ssh_user")
ssh_key = context.config.get("hg_ssh_keyfile")
ssh_opt = []
if ssh_username or ssh_key:
ssh_opt = ['-e', 'ssh']
if ssh_username:
ssh_opt[1] += ' -l %s' % ssh_username
if ssh_key:
ssh_opt[1] += ' -i %s' % ssh_key
log.info("Pushing local changes to {}".format(dest_repo_ssh))
await run_hg_command(context, 'push', *ssh_opt, '-r', '.', '-v',
dest_repo_ssh, local_repo=local_repo)
async def push(context):
"""Run `hg push` against the current source repo."""
local_repo = context.repo
dest_repo = get_source_repo(context.task)
dest_repo_ssh = dest_repo.replace('https://', 'ssh://')
ssh_username = context.config.get("hg_ssh_user")
ssh_key = context.config.get("hg_ssh_keyfile")
ssh_opt = []
if ssh_username or ssh_key:
ssh_opt = ['-e', 'ssh']
if ssh_username:
ssh_opt[1] += ' -l %s' % ssh_username
if ssh_key:
ssh_opt[1] += ' -i %s' % ssh_key
log.info("Pushing local changes to {}".format(dest_repo_ssh))
await run_hg_command(context, 'push', *ssh_opt, '-r', '.', '-v',
dest_repo_ssh, local_repo=local_repo)
async def do_actions(config, task, actions, repo_path):
"""Perform the set of actions that treescript can perform.
The actions happen in order, tagging, ver bump, then push
Args:
config (dict): the running config
task (dict): the running task
actions (list): the actions to perform
repo_path (str): the source directory to use.
"""
await checkout_repo(config, task, repo_path)
# Split the action selection up due to complexity in do_actions
# caused by different push behaviour, and action return values.
if "merge_day" in actions:
await perform_merge_actions(config, task, actions, repo_path)
return
num_changes = 0
if "tag" in actions:
num_changes += await do_tagging(config, task, repo_path)
if "version_bump" in actions:
num_changes += await bump_version(config, task, repo_path)
if "l10n_bump" in actions:
num_changes += await l10n_bump(config, task, repo_path)
num_outgoing = await log_outgoing(config, task, repo_path)
if num_outgoing != num_changes:
raise TreeScriptError(
"Outgoing changesets don't match number of expected changesets!"
" {} vs {}".format(num_outgoing, num_changes))
if should_push(task, actions):
if num_changes:
await push(config,
task,
repo_path,
target_repo=get_source_repo(task))
else:
log.info("No changes; skipping push.")
await strip_outgoing(config, task, repo_path)
async def push(config, task, repo_path):
"""Run `hg push` against the current source repo.
Args:
config (dict): the running config
task (dict): the running task
repo_path (str): the source repo path
Raises:
PushError: on failure
"""
dest_repo = get_source_repo(task)
dest_repo_ssh = dest_repo.replace("https://", "ssh://")
ssh_username = config.get("hg_ssh_user")
ssh_key = config.get("hg_ssh_keyfile")
ssh_opt = []
if ssh_username or ssh_key:
ssh_opt = ["-e", "ssh"]
if ssh_username:
ssh_opt[1] += " -l %s" % ssh_username
if ssh_key:
ssh_opt[1] += " -i %s" % ssh_key
log.info("Pushing local changes to {}".format(dest_repo_ssh))
try:
await run_hg_command(
config,
"push",
*ssh_opt,
"-r",
".",
"-v",
dest_repo_ssh,
repo_path=repo_path,
exception=PushError,
)
except PushError as exc:
log.warning("Hit PushError %s", str(exc))
await strip_outgoing(config, task, repo_path)
raise
async def assert_outgoing(context, directory, expected_cset_count):
"""Run `hg out` to ensure the expected number of changes exist.
Assuming that each action produces one changeset, we can
check to see if we'd push to the right location by comparing
the results of 'hg out' - if it's not equal to the number
of actions, we have unexpected changes to push.
"""
local_repo = os.path.join(directory, 'src')
dest_repo = get_source_repo(context.task)
log.info("outgoing changesets..")
changesets = await run_hg_command(context,
'out',
'-q',
'-r',
'.',
dest_repo,
local_repo=local_repo)
if len(changesets) != expected_cset_count:
raise ChangesetMismatchError(
'Expected {} changesets to push, found {}'.format(
expected_cset_count, changesets))
async def do_tagging(context, directory):
"""Perform tagging, at ${directory}/src.
This function will perform a mercurial tag, on 'default' head of target repository.
It will tag the revision specified in the tag_info portion of the task payload, using
the specified tags in that payload.
Tags are forced to be created at the specified revision if they already existed.
This function has the side affect of pulling the specified revision from the
destination repository. This feature exists because mozilla-unified does not
contain relbranches, though some releases are created on relbranches, so we must ensure
the desired revision to tag is known to the local repository.
Args:
context (TreeScriptContext): the treescript context
directory (str): The directory to place the resulting clone.
Raises:
FailedSubprocess: if the tag attempt doesn't succeed.
"""
local_repo = os.path.join(directory, 'src')
tag_info = get_tag_info(context.task)
desired_tags = tag_info['tags']
desired_rev = tag_info['revision']
dest_repo = get_source_repo(context.task)
commit_msg = TAG_MSG.format(revision=desired_rev, tags=', '.join(desired_tags))
log.info("Pulling {revision} from {repo} explicitly.".format(
revision=desired_rev, repo=dest_repo))
await run_hg_command(context, 'pull', '-r', desired_rev, dest_repo,
local_repo=local_repo)
log.info(commit_msg)
await run_hg_command(context, 'tag', '-m', commit_msg, '-r', desired_rev,
'-f', # Todo only force if needed
*desired_tags,
local_repo=local_repo)
def test_get_source_repo(task_defn, source_repo):
task_defn['metadata']['source'] = "{}/file/default/taskcluster/ci/foobar".format(source_repo)
assert source_repo == stask.get_source_repo(task_defn)
def test_get_source_repo_raises(task_defn, source_url, raises):
task_defn['metadata']['source'] = source_url
with pytest.raises(raises):
stask.get_source_repo(task_defn)
def test_get_payload_source_repo(task_defn, source_repo):
task_defn["payload"]["source_repo"] = source_repo
assert source_repo == ttask.get_source_repo(task_defn)
def test_get_metadata_source_repo(task_defn, source_repo):
task_defn["metadata"][
"source"] = "{}/file/default/taskcluster/ci/foobar".format(source_repo)
assert source_repo == ttask.get_source_repo(task_defn)
def test_get_source_repo_raises(task_defn, source_url, raises):
task_defn["metadata"]["source"] = source_url
with pytest.raises(raises):
ttask.get_source_repo(task_defn)
def test_get_metadata_source_repo(task_defn, source, expected_results):
task_defn["metadata"]["source"] = source
assert ttask.get_source_repo(task_defn) == expected_results
