def prep_job(job_config, log_file_path, archive_dir):
job_id = job_config['job_id']
safe_archive = safepath.munge(job_config['name'])
job_config['worker_log'] = log_file_path
archive_path_full = os.path.join(archive_dir, safe_archive, str(job_id))
job_config['archive_path'] = archive_path_full
# If the teuthology branch was not specified, default to master and
# store that value.
teuthology_branch = job_config.get('teuthology_branch', 'master')
job_config['teuthology_branch'] = teuthology_branch
teuthology_sha1 = job_config.get('teuthology_sha1')
if not teuthology_sha1:
repo_url = build_git_url('teuthology', 'ceph')
teuthology_sha1 = ls_remote(repo_url, teuthology_branch)
if not teuthology_sha1:
reason = "Teuthology branch {} not found; marking job as dead".format(
teuthology_branch)
log.error(reason)
report.try_push_job_info(
job_config, dict(status='dead', failure_reason=reason))
raise SkipJob()
log.info('Using teuthology sha1 %s', teuthology_sha1)
try:
if teuth_config.teuthology_path is not None:
teuth_path = teuth_config.teuthology_path
else:
teuth_path = fetch_teuthology(branch=teuthology_branch,
commit=teuthology_sha1)
# For the teuthology tasks, we look for suite_branch, and if we
# don't get that, we look for branch, and fall back to 'master'.
# last-in-suite jobs don't have suite_branch or branch set.
ceph_branch = job_config.get('branch', 'master')
suite_branch = job_config.get('suite_branch', ceph_branch)
suite_sha1 = job_config.get('suite_sha1')
suite_repo = job_config.get('suite_repo')
if suite_repo:
teuth_config.ceph_qa_suite_git_url = suite_repo
job_config['suite_path'] = os.path.normpath(
os.path.join(
fetch_qa_suite(suite_branch, suite_sha1),
job_config.get('suite_relpath', ''),
))
except (BranchNotFoundError, CommitNotFoundError) as exc:
log.exception("Requested version not found; marking job as dead")
report.try_push_job_info(job_config,
dict(status='dead', failure_reason=str(exc)))
raise SkipJob()
except MaxWhileTries as exc:
log.exception("Failed to fetch or bootstrap; marking job as dead")
report.try_push_job_info(job_config,
dict(status='dead', failure_reason=str(exc)))
raise SkipJob()
teuth_bin_path = os.path.join(teuth_path, 'virtualenv', 'bin')
if not os.path.isdir(teuth_bin_path):
raise RuntimeError("teuthology branch %s at %s not bootstrapped!" %
(teuthology_branch, teuth_bin_path))
return job_config, teuth_bin_path
def choose_teuthology_branch(self):
"""Select teuthology branch, check if it is present in repo and return
tuple (branch, hash) where hash is commit sha1 corresponding
to the HEAD of the branch.
The branch name value is determined in the following order:
Use ``--teuthology-branch`` argument value if supplied.
Use ``TEUTH_BRANCH`` environment variable value if declared.
If file ``qa/.teuthology_branch`` can be found in the suite repo
supplied with ``--suite-repo`` or ``--suite-dir`` and contains
non-empty string then use it as the branch name.
Use ``teuthology_branch`` value if it is set in the one
of the teuthology config files ``$HOME/teuthology.yaml``
or ``/etc/teuthology.yaml`` correspondingly.
Use ``master``.
Generate exception if the branch is not present in the repo.
"""
teuthology_branch = self.args.teuthology_branch
if not teuthology_branch:
teuthology_branch = os.environ.get('TEUTH_BRANCH', None)
if not teuthology_branch:
branch_file_path = self.suite_repo_path + '/qa/.teuthology_branch'
log.debug('Check file %s exists', branch_file_path)
if os.path.exists(branch_file_path):
log.debug('Found teuthology branch config file %s',
branch_file_path)
with open(branch_file_path) as f:
teuthology_branch = f.read().strip()
if teuthology_branch:
log.debug(
'The teuthology branch is overridden with %s',
teuthology_branch)
else:
log.warning(
'The teuthology branch config is empty, skipping')
if not teuthology_branch:
teuthology_branch = config.get('teuthology_branch', 'master')
teuthology_sha1 = util.git_ls_remote(
'teuthology',
teuthology_branch
)
if not teuthology_sha1:
exc = BranchNotFoundError(teuthology_branch, build_git_url('teuthology'))
util.schedule_fail(message=str(exc), name=self.name)
log.info("teuthology branch: %s %s", teuthology_branch, teuthology_sha1)
return teuthology_branch, teuthology_sha1
def git_ls_remote(project_or_url, branch, project_owner='ceph'):
"""
Find the latest sha1 for a given project's branch.
:param project_or_url: Either a project name or a full URL
:param branch: The branch to query
:param project_owner: The GitHub project owner. Only used when a project
name is passed; not when a URL is passed
:returns: The sha1 if found; else None
"""
if '://' in project_or_url:
url = project_or_url
else:
url = build_git_url(project_or_url, project_owner)
return repo_utils.ls_remote(url, branch)
def choose_teuthology_branch(self):
teuthology_branch = self.args.teuthology_branch
if teuthology_branch and teuthology_branch != 'master':
if not util.git_branch_exists('teuthology', teuthology_branch):
exc = BranchNotFoundError(teuthology_branch, 'teuthology.git')
util.schedule_fail(message=str(exc), name=self.name)
elif not teuthology_branch:
# Decide what branch of teuthology to use
if util.git_branch_exists('teuthology', self.args.ceph_branch):
teuthology_branch = self.args.ceph_branch
else:
log.info(
"branch {0} not in teuthology.git; will use master for"
" teuthology".format(self.args.ceph_branch))
teuthology_branch = 'master'
teuthology_hash = util.git_ls_remote('teuthology', teuthology_branch)
if not teuthology_hash:
exc = BranchNotFoundError(teuthology_branch,
build_git_url('teuthology'))
util.schedule_fail(message=str(exc), name=self.name)
log.info("teuthology branch: %s %s", teuthology_branch,
teuthology_hash)
return teuthology_branch
def git_validate_sha1(project, sha1, project_owner='ceph'):
'''
Use http to validate that project contains sha1
I can't find a way to do this with git, period, so
we have specific urls to HEAD for github and git.ceph.com/gitweb
for now
'''
url = build_git_url(project, project_owner)
if '/github.com/' in url:
url = '/'.join((url, 'commit', sha1))
elif '/git.ceph.com/' in url:
# kinda specific to knowing git.ceph.com is gitweb
url = (
'http://git.ceph.com/?p=%s.git;a=blob_plain;f=.gitignore;hb=%s' %
(project, sha1))
else:
raise RuntimeError('git_validate_sha1: how do I check %s for a sha1?' %
url)
resp = requests.head(url)
if resp.ok:
return sha1
return None
def _tag_to_sha1(self):
"""
Shaman doesn't know about tags. Use git ls-remote to query the remote
repo in order to map tags to their sha1 value.
This method will also retry against ceph.git if the original request
uses ceph-ci.git and fails.
"""
def get_sha1(url):
# Ceph (and other projects) uses annotated tags for releases. This
# has the side-effect of making git ls-remote return the sha1 for
# the annotated tag object and not the last "real" commit in that
# tag. By contrast, when a person (or a build system) issues a
# "git checkout <tag>" command, HEAD will be the last "real" commit
# and not the tag.
# Below we have to append "^{}" to the tag value to work around
# this in order to query for the sha1 that the build system uses.
return repo_utils.ls_remote(url, "%s^{}" % self.tag)
git_url = repo_utils.build_git_url(self.project)
result = get_sha1(git_url)
# For upgrade tests that are otherwise using ceph-ci.git, we need to
# also look in ceph.git to lookup released tags.
if result is None and 'ceph-ci' in git_url:
alt_git_url = git_url.replace('ceph-ci', 'ceph')
log.info(
"Tag '%s' not found in %s; will also look in %s",
self.tag,
git_url,
alt_git_url,
)
result = get_sha1(alt_git_url)
if result is None:
raise CommitNotFoundError(self.tag, git_url)
return result
