def push_tag(self):
if not self.tag:
raise ValueError('Must run set_nvr() before calling!')
with Dir(self.source_path):
exectools.cmd_assert('git push origin {}'.format(self.tag),
retries=3)
def tito_setup(self):
if self.config.content.build.use_source_tito_config:
return # rely on tito already set up in source
tito_dir = os.path.join(self.source_path, '.tito')
tito_target = self.config.content.build.tito_target
tito_target = tito_target if tito_target else 'aos'
tito_dist = self.config.content.build.tito_dist
tito_dist = tito_dist if tito_dist else '.el7aos'
with Dir(self.source_path):
if not os.path.isdir(tito_dir):
exectools.cmd_assert('tito init')
# roll the init changes into the others
exectools.cmd_assert('git reset HEAD~')
with open(os.path.join(tito_dir, 'releasers.conf'), 'w') as r:
r.write(
RELEASERS_CONF.format(
branch=self.runtime.branch,
name=self.name,
target=tito_target,
dist=tito_dist,
))
r.flush()
# fix for tito 0.6.10 which looks like remote_git_name in wrong place
with open(os.path.join(tito_dir, 'tito.props'), 'a') as props:
props.write(
TITO_PROPS.format(name=self.name, target=tito_target))
props.flush()
def commit(self, msg):
"""
Commit outstanding metadata config changes
"""
self.runtime.logger.info('Commit config: {}'.format(msg))
with Dir(self.runtime.metadata_dir):
exectools.cmd_assert(["git", "add", "."])
exectools.cmd_assert(["git", "commit", "--allow-empty", "-m", msg])
def push(self):
"""
Push changes back to config repo.
Will of course fail if user does not have write access.
"""
self.runtime.logger.info('Pushing config...')
with Dir(self.runtime.metadata_dir):
exectools.cmd_assert(["git", "push"])
def create_tag(self, scratch):
if not self.tag:
raise ValueError('Must run set_nvr() before calling!')
with Dir(self.source_path):
if not scratch:
exectools.cmd_assert('git tag {}'.format(self.tag))
rc, sha, err = exectools.cmd_gather('git rev-parse HEAD')
self.commit_sha = sha.strip()
def test_cmd_assert_fail(self):
"""
"""
# Try a failing command 3 times, at 1 sec intervals
with self.assertRaises(IOError):
exectools.cmd_assert("/usr/bin/false", 3, 1)
# check that the log file has all of the tests.
log_file = open(self.test_file, 'r')
lines = log_file.readlines()
log_file.close()
self.assertEquals(len(lines), 12)
def test_cmd_assert_success(self):
"""
"""
try:
exectools.cmd_assert("/bin/true")
except IOError as error:
self.Fail("/bin/truereturned failure: {}".format(error))
# check that the log file has all of the tests.
log_file = open(self.test_file, 'r')
lines = log_file.readlines()
log_file.close()
self.assertEquals(len(lines), 4)
def tito_setup(self):
tito_dir = os.path.join(self.source_path, '.tito')
with Dir(self.source_path):
if not os.path.isdir(tito_dir):
exectools.cmd_assert('tito init')
with open(os.path.join(tito_dir, 'releasers.conf'), 'w') as r:
r.write(RELEASERS_CONF.format(branch=self.runtime.branch,
name=self.name))
r.flush()
# fix for tito 0.6.10 which looks like remote_git_name in wrong place
with open(os.path.join(tito_dir, 'tito.props'), 'a') as props:
props.write(TITO_PROPS.format(name=self.name))
props.flush()
def builds_for_group_branch(self):
# return a dict of all the latest builds for this group, according to
# the branch's candidate tag in brew. each entry is name => tuple(version, release).
output, _ = exectools.cmd_assert(
"brew list-tagged --quiet --latest {}-candidate".format(
self.branch),
retries=3,
)
builds = [
# each line like "build tag owner" split into build NVR
line.split()[0].rsplit("-", 2)
for line in output.strip().split("\n") if line.strip()
]
return {n: (v, r) for n, v, r in builds}
def commit_changes(self, scratch):
if not self.tag:
raise ValueError('Must run set_nvr() before calling!')
with Dir(self.source_path):
if self.config.content.build.use_source_tito_config:
# just use the tito tagger to change spec and tag
exectools.cmd_assert([
"tito", "tag", "--no-auto-changelog", "--use-version",
self.version, "--use-release",
"{}%{{?dist}}".format(self.release)
])
if self.config.content.build.push_release_commit and not scratch:
exectools.cmd_assert("git push origin")
return
exectools.cmd_assert(
["git", "tag", "-am", "Release with doozer", self.tag])
exectools.cmd_assert("git add .")
commit_msg = "Automatic commit of package [{name}] release [{version}-{release}].".format(
name=self.config.name,
version=self.version,
release=self.release)
exectools.cmd_assert(['git', 'commit', '-m', commit_msg])
def _get_remote_branch_ref(self, git_url, branch):
"""Detect whether a single branch exists on a remote repo; returns git hash if found"""
self.logger.info('Checking if target branch {} exists in {}'.format(
branch, git_url))
try:
out, _ = exectools.cmd_assert('git ls-remote --heads {} {}'.format(
git_url, branch),
retries=3)
except Exception as err:
self.logger.error(
'Unable to check if target branch {} exists: {}'.format(
branch, err))
return None
result = out.strip() # any result means the branch is found
return result.split()[0] if result else None
def post_build(self, scratch):
build_spec = self.config.content.build
with Dir(self.source_path):
if self.build_status and not scratch:
# success; push the tag
try:
self.push_tag()
except Exception:
raise RuntimeError(
'Build succeeded but failure pushing RPM tag for {}'.
format(self.qualified_name))
elif build_spec.use_source_tito_config:
# don't leave the tag/commit lying around if not a valid build
exectools.cmd_assert("tito tag --undo")
if not build_spec.push_release_commit and not build_spec.use_source_tito_config:
# temporary tag/commit; never leave lying around
exectools.cmd_assert("git reset --hard HEAD~")
exectools.cmd_assert("git tag -d {}".format(self.tag))
def resolve_source(self, parent, meta):
"""
Looks up a source alias and returns a path to the directory containing
that source. Sources can be specified on the command line, or, failing
that, in group.yml.
If a source specified in group.yaml has not be resolved before,
this method will clone that source to checkout the group's desired
branch before returning a path to the cloned repo.
:param parent: Name of parent the source belongs to
:param meta: The MetaData object to resolve source for
:return: Returns the source path
"""
source = meta.config.content.source
# This allows passing `--source <distgit_key> path` to
# override any source to something local without it
# having been configured for an alias
if self.local and meta.distgit_key in self.source_paths:
source['alias'] = meta.distgit_key
if 'git' in source:
del source['git']
source_details = None
if 'git' in source:
git_url = urlparse.urlparse(source.git.url)
name = os.path.splitext(os.path.basename(git_url.path))[0]
alias = '{}_{}'.format(parent, name)
source_details = dict(source.git)
elif 'alias' in source:
alias = source.alias
else:
raise DoozerFatalError(
'Error while processing source for {}'.format(parent))
self.logger.debug(
"Resolving local source directory for alias {}".format(alias))
if alias in self.source_paths:
self.logger.debug(
"returning previously resolved path for alias {}: {}".format(
alias, self.source_paths[alias]))
return self.source_paths[alias]
# Where the source will land, check early so we know if old or new style
sub_path = '{}{}'.format('global_' if source_details is None else '',
alias)
source_dir = os.path.join(self.sources_dir, sub_path)
if not source_details: # old style alias was given
if (self.group_config.sources is Missing
or alias not in self.group_config.sources):
raise DoozerFatalError(
"Source alias not found in specified sources or in the current group: %s"
% alias)
source_details = self.group_config.sources[alias]
self.logger.debug(
"checking for source directory in source_dir: {}".format(
source_dir))
# If this source has already been extracted for this working directory
if os.path.isdir(source_dir):
# Store so that the next attempt to resolve the source hits the map
self.source_paths[alias] = source_dir
self.logger.info(
"Source '{}' already exists in (skipping clone): {}".format(
alias, source_dir))
return source_dir
clone_branch, _ = self.detect_remote_source_branch(source_details)
url = source_details["url"]
try:
self.logger.info(
"Attempting to checkout source '%s' branch %s in: %s" %
(url, clone_branch, source_dir))
exectools.cmd_assert(
# get a little history to enable finding a recent Dockerfile change, but not too much.
"git clone -b {} --single-branch {} --depth 50 {}".format(
clone_branch, url, source_dir),
retries=3,
on_retry=["rm", "-rf", source_dir],
)
except IOError as e:
self.logger.info("Unable to checkout branch {}: {}".format(
clone_branch, e.message))
raise DoozerFatalError(
"Error checking out target branch of source '%s' in: %s" %
(alias, source_dir))
# Store so that the next attempt to resolve the source hits the map
self.register_source_alias(alias, source_dir)
return source_dir
def resolve_metadata(self):
"""
The group control data can be on a local filesystem, in a git
repository that can be checked out, or some day in a database
If the scheme is empty, assume file:///...
Allow http, https, ssh and ssh+git (all valid git clone URLs)
"""
if self.metadata_dir is None:
self.metadata_dir = constants.OCP_BUILD_DATA_RW
schemes = ['ssh', 'ssh+git', "http", "https"]
self.logger.info('Using {} for metadata'.format(self.metadata_dir))
md_url = urlparse.urlparse(self.metadata_dir)
if md_url.scheme in schemes or (md_url.scheme == ''
and ':' in md_url.path):
# Assume this is a git repo to clone
#
# An empty scheme with a colon in the path is likely an "scp" style
# path: ala [email protected]:owner/path
# determine where to put it
md_name = os.path.splitext(os.path.basename(md_url.path))[0]
md_destination = os.path.join(self.working_dir, md_name)
clone_data = True
if os.path.isdir(md_destination):
self.logger.info(
'Metadata clone directory already exists, checking commit sha'
)
with Dir(md_destination):
rc, out, err = exectools.cmd_gather(
["git", "ls-remote", self.metadata_dir, "HEAD"])
if rc:
raise IOError(
'Unable to check remote sha: {}'.format(err))
remote = out.strip().split('\t')[0]
try:
exectools.cmd_assert(
'git branch --contains {}'.format(remote))
self.logger.info(
'{} is already cloned and latest'.format(
self.metadata_dir))
clone_data = False
except:
rc, out, err = exectools.cmd_gather(
'git log origin/HEAD..HEAD')
out = out.strip()
if len(out):
msg = """
Local config is out of sync with remote and you have unpushed commits. {}
You must either clear your local config repo with `./oit.py cleanup`
or manually rebase from latest remote to continue
""".format(md_destination)
raise IOError(msg)
if clone_data:
if os.path.isdir(md_destination): # delete if already there
shutil.rmtree(md_destination)
self.logger.info('Cloning config data from {}'.format(
self.metadata_dir))
if not os.path.isdir(md_destination):
cmd = "git clone --depth 1 {} {}".format(
self.metadata_dir, md_destination)
try:
exectools.cmd_assert(cmd.split(' '))
except:
if self.metadata_dir == constants.OCP_BUILD_DATA_RW:
self.logger.warn(
'Failed to clone {}, falling back to {}'.
format(constants.OCP_BUILD_DATA_RW,
constants.OCP_BUILD_DATA_RO))
self.metadata_dir = constants.OCP_BUILD_DATA_RO
return self.resolve_metadata()
else:
raise
self.metadata_dir = md_destination
elif md_url.scheme in ['', 'file']:
# no scheme, assume the path is a local file
self.metadata_dir = md_url.path
if not os.path.isdir(self.metadata_dir):
raise ValueError(
"Invalid metadata_dir: {} - Not a directory".format(
self.metadata_dir))
else:
# invalid scheme: not '' or any of the valid list
raise ValueError(
"Invalid metadata_dir: {} - invalid scheme: {}".format(
self.metadata_dir, md_url.scheme))
def resolve_source(self, alias, required=True):
"""
Looks up a source alias and returns a path to the directory containing
that source. Sources can be specified on the command line, or, failing
that, in group.yml.
If a source specified in group.yaml has not be resolved before,
this method will clone that source to checkout the group's desired
branch before returning a path to the cloned repo.
:param alias: The source alias to resolve
:param required: If True, thrown an exception if not found
:return: Returns the source path or None (if required=False)
"""
self.logger.debug(
"Resolving local source directory for alias {}".format(alias))
if alias in self.source_paths:
self.logger.debug(
"returning previously resolved path for alias {}: {}".format(
alias, self.source_paths[alias]))
return self.source_paths[alias]
# Check if the group config specs the "alias" for the source location
if (self.group_config.sources is Missing
or alias not in self.group_config.sources):
if required:
raise IOError(
"Source alias not found in specified sources or in the current group: %s"
% alias)
else:
return None
# Where the source will land
source_dir = os.path.join(self.sources_dir, alias)
self.logger.debug(
"checking for source directory in source_dir: {}".format(
source_dir))
# If this source has already been extracted for this working directory
if os.path.isdir(source_dir):
# Store so that the next attempt to resolve the source hits the map
self.source_paths[alias] = source_dir
self.logger.info(
"Source '{}' already exists in (skipping clone): {}".format(
alias, source_dir))
return source_dir
source_config = self.group_config.sources[alias]
url = source_config["url"]
branches = source_config['branch']
self.logger.info(
"Cloning source '%s' from %s as specified by group into: %s" %
(alias, url, source_dir))
exectools.cmd_assert(
cmd=["git", "clone", url, source_dir],
retries=3,
on_retry=["rm", "-rf", source_dir],
)
stage_branch = branches.get('stage', None)
fallback_branch = branches.get("fallback", None)
found = False
with Dir(source_dir):
if self.stage and stage_branch:
self.logger.info(
'Normal branch overridden by --stage option, using "{}"'.
format(stage_branch))
branch = stage_branch
else:
branch = branches["target"]
self.logger.info(
"Attempting to checkout source '%s' branch %s in: %s" %
(alias, branch, source_dir))
if branch != "master":
rc, out, err = exectools.cmd_gather(
["git", "checkout", "-b", branch,
"origin/%s" % branch])
else:
rc = 0
if rc == 0:
found = True
else:
if self.stage and stage_branch:
raise IOError(
'--stage option specified and no stage branch named "{}" exists for {}|{}'
.format(stage_branch, alias, url))
elif fallback_branch is not None:
self.logger.info(
"Unable to checkout branch %s ; trying fallback %s" %
(branch, fallback_branch))
self.logger.info(
"Attempting to checkout source '%s' fallback-branch %s in: %s"
% (alias, fallback_branch, source_dir))
if fallback_branch != "master":
rc2, out, err = exectools.cmd_gather([
"git", "checkout", "-b", fallback_branch,
"origin/%s" % fallback_branch
], )
else:
rc2 = 0
if rc2 == 0:
found = True
else:
self.logger.error(
"Failed checking out fallback-branch %s: %s" %
(branch, err))
else:
self.logger.error("Failed checking out branch %s: %s" %
(branch, err))
if found:
# Store so that the next attempt to resolve the source hits the map
self.register_source_alias(alias, source_dir)
return source_dir
else:
if required:
raise IOError(
"Error checking out target branch of source '%s' in: %s"
% (alias, source_dir))
else:
return None
def commit_changes(self):
with Dir(self.source_path):
exectools.cmd_assert("git add .")
exectools.cmd_assert(
['git', 'commit', '-m', "Local commit for dist-git build"])
def update_spec(self):
if self.config.content.build.use_source_tito_config:
# tito tag will handle updating specfile
return
# otherwise, make changes similar to tito tagging
replace = {
'Name:': 'Name: {}\n'.format(self.config.name),
'Version:': 'Version: {}\n'.format(self.version),
'Release:': 'Release: {}%{{?dist}}\n'.format(self.release),
}
# self.version example: 3.9.0
# Extract the major, minor, patch
major, minor, patch = self.version.split('.')
full = "v{}".format(self.version)
# If this is a pre-release RPM, the include the release field in
# the full version.
# pre-release full version: v3.9.0-0.20.1
# release full version: v3.9.0
if self.release.startswith("0."):
full += "-{}".format(self.release)
replace_keys = replace.keys()
with Dir(self.source_path):
commit_sha = exectools.cmd_assert('git rev-parse HEAD')[0].strip()
# run generic modifications first
if self.config.content.source.modifications is not Missing:
self._run_modifications()
# second, update with NVR
with open(self.specfile, 'r+') as sf:
lines = sf.readlines()
for i in range(len(lines)):
if "%global os_git_vars " in lines[i]:
lines[
i] = "%global os_git_vars OS_GIT_VERSION={version} OS_GIT_MAJOR={major} OS_GIT_MINOR={minor} OS_GIT_PATCH={patch} OS_GIT_COMMIT={commit} OS_GIT_TREE_STATE=clean\n".format(
version=full,
major=major,
minor=minor,
patch=patch,
commit=commit_sha)
elif "%global commit" in lines[i]:
lines[i] = re.sub(r'commit\s+\w+',
"commit {}".format(commit_sha),
lines[i])
elif replace_keys: # If there are keys left to replace
for k in replace_keys:
v = replace[k]
if lines[i].startswith(k):
lines[i] = v
replace_keys.remove(k)
break
# truncate the original file
sf.seek(0)
sf.truncate()
# write back new lines
sf.writelines(lines)
