def choose_ceph_hash(self):
"""
Get the ceph hash: if --sha1/-S is supplied, use it if it is valid, and
just keep the ceph_branch around. Otherwise use the current git branch
tip.
"""
repo_name = self.ceph_repo_name
if self.args.ceph_sha1:
ceph_hash = self.args.ceph_sha1
if self.args.validate_sha1:
ceph_hash = util.git_validate_sha1(repo_name, ceph_hash)
if not ceph_hash:
exc = CommitNotFoundError(self.args.ceph_sha1,
'%s.git' % repo_name)
util.schedule_fail(message=str(exc), name=self.name)
log.info("ceph sha1 explicitly supplied")
elif self.args.ceph_branch:
ceph_hash = util.git_ls_remote(repo_name, self.args.ceph_branch)
if not ceph_hash:
exc = BranchNotFoundError(self.args.ceph_branch,
'%s.git' % repo_name)
util.schedule_fail(message=str(exc), name=self.name)
log.info("ceph sha1: {hash}".format(hash=ceph_hash))
return ceph_hash
def choose_suite_branch(self):
suite_repo_name = self.suite_repo_name
suite_repo_project_or_url = self.args.suite_repo or 'ceph-qa-suite'
suite_branch = self.args.suite_branch
ceph_branch = self.args.ceph_branch
if suite_branch and suite_branch != 'master':
if not util.git_branch_exists(
suite_repo_project_or_url,
suite_branch
):
exc = BranchNotFoundError(suite_branch, suite_repo_name)
util.schedule_fail(message=str(exc), name=self.name)
elif not suite_branch:
# Decide what branch of the suite repo to use
if util.git_branch_exists(suite_repo_project_or_url, ceph_branch):
suite_branch = ceph_branch
else:
log.info(
"branch {0} not in {1}; will use master for"
" ceph-qa-suite".format(
ceph_branch,
suite_repo_name
))
suite_branch = 'master'
return suite_branch
def fetch_branch(repo_path, branch, shallow=True):
"""
Call "git fetch -p origin <branch>"
:param repo_path: The full path to the repository on-disk
:param branch: The branch.
:param shallow: Whether to perform a shallow fetch (--depth 1)
:raises: BranchNotFoundError if the branch is not found;
GitError for other errors
"""
validate_branch(branch)
log.info("Fetching %s from origin", branch)
args = ['git', 'fetch']
if shallow:
args.extend(['--depth', '1'])
args.extend(['-p', 'origin', fetch_refspec(branch)])
proc = subprocess.Popen(args,
cwd=repo_path,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if proc.wait() != 0:
not_found_str = "fatal: couldn't find remote ref %s" % branch
out = proc.stdout.read().decode()
log.error(out)
if not_found_str in out.lower():
raise BranchNotFoundError(branch)
else:
raise GitError("git fetch failed!")
def reset_repo(repo_url, dest_path, branch, commit=None):
"""
:param repo_url: The full URL to the repo (not including the branch)
:param dest_path: The full path to the destination directory
:param branch: The branch.
:param commit: The sha1 to checkout. Defaults to None, which uses HEAD of the branch.
:raises: BranchNotFoundError if the branch is not found;
CommitNotFoundError if the commit is not found;
GitError for other errors
"""
validate_branch(branch)
if '/' in branch:
reset_branch = lsstrip(remote_ref_from_ref(branch), 'refs/remotes/')
else:
reset_branch = 'origin/%s' % branch
reset_ref = commit or reset_branch
log.info('Resetting repo at %s to %s', dest_path, reset_ref)
# This try/except block will notice if the requested branch doesn't
# exist, whether it was cloned or fetched.
try:
subprocess.check_output(
('git', 'reset', '--hard', reset_ref),
cwd=dest_path,
)
except subprocess.CalledProcessError:
if commit:
raise CommitNotFoundError(commit, repo_url)
raise BranchNotFoundError(branch, repo_url)
def choose_suite_hash(self, suite_branch):
suite_repo_name = self.suite_repo_name
suite_repo_project_or_url = self.args.suite_repo or 'ceph-qa-suite'
suite_hash = util.git_ls_remote(suite_repo_project_or_url,
suite_branch)
if not suite_hash:
exc = BranchNotFoundError(suite_branch, suite_repo_name)
util.schedule_fail(message=str(exc), name=self.name)
log.info("%s branch: %s %s", suite_repo_name, suite_branch, suite_hash)
return suite_hash
def choose_teuthology_branch(self):
"""Select teuthology branch, check if it is present in repo and return
tuple (branch, hash) where hash is commit sha1 corresponding
to the HEAD of the branch.
The branch name value is determined in the following order:
Use ``--teuthology-branch`` argument value if supplied.
Use ``TEUTH_BRANCH`` environment variable value if declared.
If file ``qa/.teuthology_branch`` can be found in the suite repo
supplied with ``--suite-repo`` or ``--suite-dir`` and contains
non-empty string then use it as the branch name.
Use ``teuthology_branch`` value if it is set in the one
of the teuthology config files ``$HOME/teuthology.yaml``
or ``/etc/teuthology.yaml`` correspondingly.
Use ``master``.
Generate exception if the branch is not present in the repo.
"""
teuthology_branch = self.args.teuthology_branch
if not teuthology_branch:
teuthology_branch = os.environ.get('TEUTH_BRANCH', None)
if not teuthology_branch:
branch_file_path = self.suite_repo_path + '/qa/.teuthology_branch'
log.debug('Check file %s exists', branch_file_path)
if os.path.exists(branch_file_path):
log.debug('Found teuthology branch config file %s',
branch_file_path)
with open(branch_file_path) as f:
teuthology_branch = f.read().strip()
if teuthology_branch:
log.debug(
'The teuthology branch is overridden with %s',
teuthology_branch)
else:
log.warning(
'The teuthology branch config is empty, skipping')
if not teuthology_branch:
teuthology_branch = config.get('teuthology_branch', 'master')
teuthology_sha1 = util.git_ls_remote(
'teuthology',
teuthology_branch
)
if not teuthology_sha1:
exc = BranchNotFoundError(teuthology_branch, build_git_url('teuthology'))
util.schedule_fail(message=str(exc), name=self.name)
log.info("teuthology branch: %s %s", teuthology_branch, teuthology_sha1)
return teuthology_branch, teuthology_sha1
def choose_teuthology_branch(self):
teuthology_branch = self.args.teuthology_branch
if teuthology_branch and teuthology_branch != 'master':
if not util.git_branch_exists('teuthology', teuthology_branch):
exc = BranchNotFoundError(teuthology_branch, 'teuthology.git')
util.schedule_fail(message=str(exc), name=self.name)
elif not teuthology_branch:
# Decide what branch of teuthology to use
if util.git_branch_exists('teuthology', self.args.ceph_branch):
teuthology_branch = self.args.ceph_branch
else:
log.info(
"branch {0} not in teuthology.git; will use master for"
" teuthology".format(self.args.ceph_branch))
teuthology_branch = 'master'
teuthology_hash = util.git_ls_remote('teuthology', teuthology_branch)
if not teuthology_hash:
exc = BranchNotFoundError(teuthology_branch,
build_git_url('teuthology'))
util.schedule_fail(message=str(exc), name=self.name)
log.info("teuthology branch: %s %s", teuthology_branch,
teuthology_hash)
return teuthology_branch
def reset_repo(repo_url, dest_path, branch):
"""
:param repo_url: The full URL to the repo (not including the branch)
:param dest_path: The full path to the destination directory
:param branch: The branch.
:raises: BranchNotFoundError if the branch is not found;
GitError for other errors
"""
validate_branch(branch)
log.info('Resetting repo at %s to branch %s', dest_path, branch)
# This try/except block will notice if the requested branch doesn't
# exist, whether it was cloned or fetched.
try:
subprocess.check_output(
('git', 'reset', '--hard', 'origin/%s' % branch),
cwd=dest_path,
)
except subprocess.CalledProcessError:
raise BranchNotFoundError(branch, repo_url)
def clone_repo(repo_url, dest_path, branch, shallow=True):
"""
Clone a repo into a path
:param repo_url: The full URL to the repo (not including the branch)
:param dest_path: The full path to the destination directory
:param branch: The branch.
:param shallow: Whether to perform a shallow clone (--depth 1)
:raises: BranchNotFoundError if the branch is not found;
GitError for other errors
"""
validate_branch(branch)
log.info("Cloning %s %s from upstream", repo_url, branch)
if branch.startswith('refs/'):
clone_repo_ref(repo_url, dest_path, branch)
return
args = ['git', 'clone']
if shallow:
args.extend(['--depth', '1'])
args.extend(['--branch', branch, repo_url, dest_path])
proc = subprocess.Popen(
args,
cwd=os.path.dirname(dest_path),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
not_found_str = "Remote branch %s not found" % branch
out = proc.stdout.read()
result = proc.wait()
# Newer git versions will bail if the branch is not found, but older ones
# will not. Fortunately they both output similar text.
if not_found_str in out:
log.error(out)
if result == 0:
# Old git left a repo with the wrong branch. Remove it.
shutil.rmtree(dest_path, ignore_errors=True)
raise BranchNotFoundError(branch, repo_url)
elif result != 0:
# Unknown error
raise GitError("git clone failed!")
