def prep_job(job_config, log_file_path, archive_dir):
job_id = job_config['job_id']
safe_archive = safepath.munge(job_config['name'])
job_config['worker_log'] = log_file_path
archive_path_full = os.path.join(archive_dir, safe_archive, str(job_id))
job_config['archive_path'] = archive_path_full
# If the teuthology branch was not specified, default to master and
# store that value.
teuthology_branch = job_config.get('teuthology_branch', 'master')
job_config['teuthology_branch'] = teuthology_branch
teuthology_sha1 = job_config.get('teuthology_sha1')
if not teuthology_sha1:
repo_url = build_git_url('teuthology', 'ceph')
teuthology_sha1 = ls_remote(repo_url, teuthology_branch)
if not teuthology_sha1:
reason = "Teuthology branch {} not found; marking job as dead".format(
teuthology_branch)
log.error(reason)
report.try_push_job_info(
job_config, dict(status='dead', failure_reason=reason))
raise SkipJob()
log.info('Using teuthology sha1 %s', teuthology_sha1)
try:
if teuth_config.teuthology_path is not None:
teuth_path = teuth_config.teuthology_path
else:
teuth_path = fetch_teuthology(branch=teuthology_branch,
commit=teuthology_sha1)
# For the teuthology tasks, we look for suite_branch, and if we
# don't get that, we look for branch, and fall back to 'master'.
# last-in-suite jobs don't have suite_branch or branch set.
ceph_branch = job_config.get('branch', 'master')
suite_branch = job_config.get('suite_branch', ceph_branch)
suite_sha1 = job_config.get('suite_sha1')
suite_repo = job_config.get('suite_repo')
if suite_repo:
teuth_config.ceph_qa_suite_git_url = suite_repo
job_config['suite_path'] = os.path.normpath(
os.path.join(
fetch_qa_suite(suite_branch, suite_sha1),
job_config.get('suite_relpath', ''),
))
except (BranchNotFoundError, CommitNotFoundError) as exc:
log.exception("Requested version not found; marking job as dead")
report.try_push_job_info(job_config,
dict(status='dead', failure_reason=str(exc)))
raise SkipJob()
except MaxWhileTries as exc:
log.exception("Failed to fetch or bootstrap; marking job as dead")
report.try_push_job_info(job_config,
dict(status='dead', failure_reason=str(exc)))
raise SkipJob()
teuth_bin_path = os.path.join(teuth_path, 'virtualenv', 'bin')
if not os.path.isdir(teuth_bin_path):
raise RuntimeError("teuthology branch %s at %s not bootstrapped!" %
(teuthology_branch, teuth_bin_path))
return job_config, teuth_bin_path
def get_sha1(url):
# Ceph (and other projects) uses annotated tags for releases. This
# has the side-effect of making git ls-remote return the sha1 for
# the annotated tag object and not the last "real" commit in that
# tag. By contrast, when a person (or a build system) issues a
# "git checkout <tag>" command, HEAD will be the last "real" commit
# and not the tag.
# Below we have to append "^{}" to the tag value to work around
# this in order to query for the sha1 that the build system uses.
return repo_utils.ls_remote(url, "%s^{}" % self.tag)
def git_ls_remote(project_or_url, branch, project_owner='ceph'):
"""
Find the latest sha1 for a given project's branch.
:param project_or_url: Either a project name or a full URL
:param branch: The branch to query
:param project_owner: The GitHub project owner. Only used when a project
name is passed; not when a URL is passed
:returns: The sha1 if found; else None
"""
if '://' in project_or_url:
url = project_or_url
else:
url = build_git_url(project_or_url, project_owner)
return repo_utils.ls_remote(url, branch)