def make_default_options():
"""Helper function for creating default options for runner."""
parser = argparse.ArgumentParser()
GitRunner.add_parser_args(parser, {'github_disable_upstream_push': True})
parser.add_argument('--output_dir',
default=os.path.join('/tmp', 'gittest.%d' % os.getpid()))
return parser.parse_args([])
def to_git_url_prefix(self, url):
"""Determine url up to the terminal path component."""
if url.startswith('git@'):
parts = GitRunner.normalize_repo_url(url)
url = GitRunner.make_https_url(*parts)
# We're assuming no query parameter/fragment since these are git URLs.
# otherwise we need to parse the url and extract the path
return url[:url.rfind('/')]
def add_parser_args(parser, defaults):
"""Add standard parser arguments used by SourceCodeManager."""
if hasattr(parser, 'added_scm'):
return
parser.added_scm = True
GitRunner.add_parser_args(parser, defaults)
add_parser_argument(parser, 'github_upstream_owner',
defaults, 'spinnaker',
help='The standard upstream repository owner.')
def main():
"""The main command dispatcher."""
start_time = time.time()
from importlib import import_module
command_modules = [
import_module(name + '_commands') for name in [
'apidocs',
'bom',
'changelog',
'container',
'debian',
'halyard',
'image',
'rpm',
'source',
'spinnaker',
'inspection',
'spin',
]]
GitRunner.stash_and_clear_auth_env_vars()
options, command_registry = init_options_and_registry(
sys.argv[1:], command_modules)
logging.basicConfig(
format='%(levelname).1s %(asctime)s.%(msecs)03d'
' [%(threadName)s.%(process)d] %(message)s',
datefmt='%H:%M:%S',
level=STANDARD_LOG_LEVELS[options.log_level])
logging.debug(
'Running with options:\n %s',
'\n '.join(yaml.safe_dump(vars(options), default_flow_style=False)
.split('\n')))
factory = command_registry.get(options.command)
if not factory:
logging.error('Unknown command "%s"', options.command)
return -1
MetricsManager.startup_metrics(options)
labels = {'command': options.command}
success = False
try:
command = factory.make_command(options)
command()
success = True
finally:
labels['success'] = success
MetricsManager.singleton().observe_timer(
'BuildTool_Outcome', labels,
time.time() - start_time)
MetricsManager.shutdown_metrics()
return 0
def add_parser_args(parser, defaults):
"""Add standard parser arguments used by SourceCodeManager."""
if hasattr(parser, 'added_branch_scm'):
return
parser.added_branch_scm = True
SpinnakerSourceCodeManager.add_parser_args(parser, defaults)
GitRunner.add_parser_args(parser, defaults)
add_parser_argument(parser, 'git_branch', defaults, None,
help='The git branch to operate on.')
add_parser_argument(parser, 'github_hostname', defaults, 'github.com',
help='The hostname of the git server.')
def init_argparser(self, parser, defaults):
"""Adds command-specific arguments."""
super(PublishApiDocsFactory, self).init_argparser(
parser, defaults)
GitRunner.add_parser_args(parser, defaults)
GitRunner.add_publishing_parser_args(parser, defaults)
self.add_argument(
parser, 'git_branch', defaults, None,
help='The branch to checkout in ' + SPINNAKER_GITHUB_IO_REPOSITORY_NAME)
self.add_argument(
parser, 'spinnaker_version', defaults, None,
help='The version of spinnaker this documentation is for.')
def init_argparser(self, parser, defaults):
super(PublishChangelogFactory, self).init_argparser(
parser, defaults)
GitRunner.add_parser_args(parser, defaults)
GitRunner.add_publishing_parser_args(parser, defaults)
self.add_argument(
parser, 'spinnaker_version', defaults, None,
help='The version of spinnaker this documentation is for.')
self.add_argument(
parser, 'changelog_gist_url', defaults, None,
help='The gist to the existing changelog content being published.')
def init_argparser(self, parser, defaults):
GitRunner.add_parser_args(parser, defaults)
GitRunner.add_publishing_parser_args(parser, defaults)
super(InitiateReleaseBranchFactory, self).init_argparser(parser, defaults)
self.add_argument(
parser, 'skip_existing', defaults, False, type=bool,
help='Leave the existing tag if found in a repository.')
self.add_argument(
parser, 'delete_existing', defaults, False, type=bool,
help='Delete the existing tag if found in a repository.')
self.add_argument(
parser, 'spinnaker_version', defaults, None,
help='The version branch name should be "release-<num>.<num>.x"')
def __init__(self, factory, options, **kwargs):
super(PublishSpinnakerCommand, self).__init__(factory, options, **kwargs)
check_options_set(options, [
'spinnaker_version',
'spinnaker_release_alias',
'bom_version',
'changelog_gist_url',
'github_owner',
'min_halyard_version'
])
major, minor, _ = self.options.spinnaker_version.split('.')
self.__branch = 'release-{major}.{minor}.x'.format(
major=major, minor=minor)
options_copy = copy.copy(options)
self.__bom_scm = BomSourceCodeManager(options_copy, self.get_input_dir())
self.__hal = HalRunner(options)
self.__git = GitRunner(options)
self.__hal.check_property(
'spinnaker.config.input.bucket', options.halyard_bom_bucket)
if options.only_repositories:
self.__only_repositories = options.only_repositories.split(',')
else:
self.__only_repositories = []
options_copy.git_branch = self.__branch
self.__branch_scm = BranchSourceCodeManager(
options_copy, self.get_input_dir())
def init_argparser(self, parser, defaults):
GitRunner.add_parser_args(parser, defaults)
GitRunner.add_publishing_parser_args(parser, defaults)
super(PublishSpinCommandFactory, self).init_argparser(
parser, defaults)
self.add_argument(
parser, 'spin_bucket', defaults, None,
help='The bucket to publish spin binaries to.')
self.add_argument(
parser, 'spin_credentials_path', defaults, None,
help='The credentials to use to authenticate with the bucket.')
self.add_argument(
parser, 'spin_version', defaults, None,
help='The semantic version of the release to publish.')
# BomSourceCodeManager adds bom_version and bom_path arguments to fetch BOMs.
BomSourceCodeManager.add_parser_args(parser, defaults)
def init_argparser(self, parser, defaults):
super(PushChangelogFactory, self).init_argparser(
parser, defaults)
GitRunner.add_parser_args(parser, defaults)
self.add_argument(
parser, 'changelog_path', defaults, None,
help='The path to the changelog to push.')
self.add_argument(
parser, 'git_branch', defaults, None,
help='The branch name that this changelog is for. Note that this does'
' not actually *use* any branches, rather the branch name is used'
' to decorates the changelog filename.')
self.add_argument(
parser, 'build_changelog_gist_url', defaults, None,
help='The gist to push the changelog into.')
def __init__(self, options, root_source_dir, **kwargs):
self.__max_threads = kwargs.pop('max_threads', 100)
self.__add_upstream = kwargs.pop('attach_upstream', False)
check_kwargs_empty(kwargs)
self.__options = options
self.__git = GitRunner(options)
self.__root_source_dir = root_source_dir
def init_argparser(self, parser, defaults):
super(PublishHalyardCommandFactory, self).init_argparser(
parser, defaults)
GradleCommandFactory.add_bom_parser_args(parser, defaults)
SpinnakerSourceCodeManager.add_parser_args(parser, defaults)
GradleRunner.add_parser_args(parser, defaults)
GitRunner.add_publishing_parser_args(parser, defaults)
HalRunner.add_parser_args(parser, defaults)
self.add_argument(
parser, 'build_number', defaults, DEFAULT_BUILD_NUMBER,
help='Publishing halyard requires a rebuild. This is the build number'
' to use when rebuilding halyard.')
self.add_argument(
parser, 'halyard_version', defaults, None,
help='The semantic version of the release to publish.')
self.add_argument(
parser, 'halyard_version_commits_url', defaults, None,
help='URL to file containing version and git commit for successful'
' nightly builds. By default this will be'
' "{filename}" in the'
' --halyard_bucket_base_url.'.format(
filename=BuildHalyardCommand.HALYARD_VERSIONS_BASENAME))
self.add_argument(
parser, 'halyard_docker_image_base',
defaults, None,
help='Base Docker image name for writing halyard builds.')
self.add_argument(
parser, 'halyard_bucket_base_url',
defaults, None,
help='Base Google Cloud Storage URL for writing halyard builds.')
self.add_argument(parser, 'docs_repo_owner', defaults, None,
help='Owner of the docs repo if one was'
' specified. The default is --github_owner.')
self.add_argument(
parser, 'skip_existing', defaults, False, type=bool,
help='Skip builds if the desired version already exists on bintray.')
self.add_argument(
parser, 'delete_existing', defaults, None, type=bool,
help='Delete pre-existing desired versions if from bintray.')
def init_argparser(self, parser, defaults):
super(PublishSpinnakerFactory, self).init_argparser(parser, defaults)
HalRunner.add_parser_args(parser, defaults)
GitRunner.add_parser_args(parser, defaults)
GitRunner.add_publishing_parser_args(parser, defaults)
PublishChangelogFactory().init_argparser(parser, defaults)
self.add_argument(
parser, 'spinnaker_release_alias', defaults, None,
help='The spinnaker version alias to publish as.')
self.add_argument(
parser, 'halyard_bom_bucket', defaults, 'halconfig',
help='The bucket manaing halyard BOMs and config profiles.')
self.add_argument(
parser, 'bom_version', defaults, None,
help='The existing bom version usef for this release.')
self.add_argument(
parser, 'min_halyard_version', defaults, None,
help='The minimum halyard version required.')
def __init__(self, factory, options, **kwargs):
super(PushChangelogCommand, self).__init__(factory, options, **kwargs)
check_options_set(
options, ['build_changelog_gist_url', 'git_branch'])
if not options.changelog_path:
options.changelog_path = os.path.join(
self.get_output_dir(command=BUILD_CHANGELOG_COMMAND), 'changelog.md')
check_path_exists(options.changelog_path, why='changelog_path')
self.__git = GitRunner(options)
def test_is_same_repo(self):
variants = [
'http://github.com/user/spinnaker',
'http://github.com/user/spinnaker.git',
'https://github.com/user/spinnaker',
'https://github.com/user/spinnaker.git',
'[email protected]:user/spinnaker.git',
'[email protected]:user/spinnaker.git'
]
for url in variants:
self.assertTrue(GitRunner.is_same_repo(variants[0], url))
def test_different_repo(self):
variants = [
'http://github.com/user/spinnaker',
'http://github.com/path/user/spinnaker',
'http://github.com/user/spinnaker/path',
'http://github.com/user/spinnaker.github',
'http://github/user/spinnaker',
'http://mydomain.com/user/spinnaker',
'path/user/spinnaker'
]
for url in variants[1:]:
self.assertFalse(GitRunner.is_same_repo(variants[0], url))
class InitiateReleaseBranchCommand(RepositoryCommandProcessor):
def __init__(self, factory, options, **kwargs):
super(InitiateReleaseBranchCommand, self).__init__(
factory, options, **kwargs)
check_options_set(options, ['spinnaker_version'])
self.__git = GitRunner(options)
def _do_repository(self, repository):
git_dir = repository.git_dir
branch = self.options.spinnaker_version
logging.debug('Checking for branch="%s" in "%s"', branch, git_dir)
remote_branches = [
line.strip()
for line in self.__git.check_run(git_dir, 'branch -r').split('\n')]
if 'origin/' + branch in remote_branches:
if self.options.skip_existing:
logging.info('Branch "%s" already exists in "%s" -- skip',
branch, repository.origin)
return
elif self.options.delete_existing:
logging.warning('Branch "%s" already exists in "%s" -- delete',
branch, repository.origin)
self.__git.delete_branch_on_origin(git_dir, branch)
else:
raise_and_log_error(
ConfigError(
'Branch "{branch}" already exists in "{repo}"'.format(
branch=branch, repo=repository.name),
cause='branch_exists'))
logging.info('Creating and pushing branch "%s" to "%s"',
branch, repository.origin)
self.__git.check_run(git_dir, 'checkout -b ' + branch)
self.__git.push_branch_to_origin(git_dir, branch)
def __init__(self, factory, options, **kwargs):
super(InitiateReleaseBranchCommand, self).__init__(
factory, options, **kwargs)
check_options_set(options, ['spinnaker_version'])
self.__git = GitRunner(options)
class SpinnakerSourceCodeManager(object):
"""Helper class for managing spinnaker source code repositories."""
AUTO = '_auto_'
@staticmethod
def add_parser_args(parser, defaults):
"""Add standard parser arguments used by SourceCodeManager."""
if hasattr(parser, 'added_scm'):
return
parser.added_scm = True
GitRunner.add_parser_args(parser, defaults)
add_parser_argument(parser, 'github_upstream_owner',
defaults, 'spinnaker',
help='The standard upstream repository owner.')
@property
def git(self):
return self.__git
@property
def options(self):
return self.__options
@property
def root_source_dir(self):
"""The base directory for all the source repositories.
Each repository will be a child directory of this path.
"""
return self.__root_source_dir
def __init__(self, options, root_source_dir, **kwargs):
self.__max_threads = kwargs.pop('max_threads', 100)
self.__add_upstream = kwargs.pop('attach_upstream', False)
check_kwargs_empty(kwargs)
self.__options = options
self.__git = GitRunner(options)
self.__root_source_dir = root_source_dir
def service_name_to_repository_name(self, service_name):
if service_name == 'monitoring-daemon':
return 'spinnaker-monitoring'
return service_name
def repository_name_to_service_name(self, repository_name):
if repository_name == 'spinnaker-monitoring':
return 'monitoring-daemon'
return repository_name
def check_repository_is_current(self, repository):
raise NotImplementedError(self.__class__.__name__)
def determine_build_number(self, repository):
raise NotImplementedError(self.__class__.__name__)
def determine_origin(self, name):
"""Determine the origin URL for the given repository name."""
raise NotImplementedError(self.__class__.__name__)
def make_repository_spec(self, name, **kwargs):
"""Create GitRepositorySpec based on the name and configuration.
Args:
git_dir: if supplied then use it, otherwise default under the root path.
origin: if supplied then use it, even if None. Otherwise default
upstream: if supplied then use it, even if None. Otherwise default.
kwargs: Additional repository attributes
"""
git_dir = kwargs.pop('git_dir', os.path.join(self.__root_source_dir, name))
origin = kwargs.pop('origin', self.AUTO)
upstream = kwargs.pop('upstream', self.AUTO)
if origin == self.AUTO:
origin = self.determine_origin(name)
if os.path.exists(git_dir):
logging.info('Confirming existing %s matches expectations', git_dir)
existing = self.__git.determine_git_repository_spec(git_dir)
if existing.origin != origin:
raise_and_log_error(
UnexpectedError(
'Repository "{dir}" origin="{have}" expected="{want}"'.format(
dir=git_dir, have=existing.origin, want=origin)))
if upstream == self.AUTO:
upstream = self.determine_upstream_url(name)
return GitRepositorySpec(
name, origin=origin, upstream=upstream, git_dir=git_dir, **kwargs)
def determine_upstream_url(self, name):
upstream_owner = (self.__options.github_upstream_owner
if name not in ('citest')
else 'google')
return 'https://github.com/{upstream}/{name}'.format(
upstream=upstream_owner, name=name)
def ensure_git_path(self, repository, **kwargs):
"""Make sure the repository is checked out.
Normally one would use ensure_repository which also ensures the metadata
is cached. However repositories that are not version controlled in the
normal way (e.g. spinnaker.github.io) dont use the metadata so the
assumptions in ensure_repository are not applicable.
Returns: The build number to use
"""
raise NotImplementedError(self.__class__.__name__)
def ensure_local_repository(self, repository, commit=None):
"""Make sure local repository directory exists, and make it so if not."""
git_dir = repository.git_dir
have_git_dir = os.path.exists(git_dir)
if have_git_dir:
self.check_repository_is_current(repository)
else:
self.ensure_git_path(repository)
def refresh_source_info(self, repository, build_number):
"""Extract the source info from repository and cache with build number.
We associate the build number because the different builds
(debian, container, etc) need to have the same builder number so that the
eventual BOM is consitent. Since we dont build everything at once, we'll
need to remember it.
We extract out the repository summary info, particularly the commit it is
at, to ensure that future operations are consistent and operating on the
same commit.
"""
summary = self.__git.collect_repository_summary(repository.git_dir)
expect_build_number = (self.__options.build_number
if hasattr(self.__options, 'build_number')
else build_number)
info = SourceInfo(expect_build_number, summary)
filename = repository.name + '-meta.yml'
dir_path = os.path.join(self.__options.output_dir, 'source_info')
cache_path = os.path.join(dir_path, filename)
logging.debug(
'Refreshing source info for %s and caching to %s for buildnum=%s',
repository.name, cache_path, build_number)
write_to_path(info.summary.to_yaml(), cache_path)
return info
def lookup_source_info(self, repository):
"""Return the SourceInfo for the given repository."""
filename = repository.name + '-meta.yml'
dir_path = os.path.join(self.__options.output_dir, 'source_info')
build_number = self.determine_build_number(repository)
with open(os.path.join(dir_path, filename), 'r') as stream:
return SourceInfo(
build_number,
RepositorySummary.from_dict(yaml.safe_load(stream.read())))
def check_source_info(self, repository):
"""Ensure cached source info is consistent with current repository."""
logging.debug('Checking that cached commit is consistent with %s',
repository.git_dir)
info = self.lookup_source_info(repository)
commit = self.__git.query_local_repository_commit_id(repository.git_dir)
cached_commit = info.summary.commit_id
if cached_commit != commit:
raise_and_log_error(
UnexpectedError(
'Cached commit {cache} != current commit {id} in {dir}'.format(
cache=cached_commit, id=commit, dir=repository.git_dir)))
return info
def foreach_source_repository(
self, all_repos, call_function, *posargs, **kwargs):
"""Call the function on each of the SourceRepository instances."""
worker = RepositoryWorker(call_function, *posargs, **kwargs)
num_threads = min(self.__max_threads, len(all_repos))
if num_threads > 1:
pool = ThreadPool(num_threads)
logging.info('Mapping %d/%s',
len(all_repos), [repo.name for repo in all_repos])
try:
raw_list = pool.map(worker, all_repos)
result = {name: value for name, value in raw_list}
except Exception:
logging.error('Map caught exception')
pool.close()
pool.join()
raise
logging.info('Finished mapping')
pool.close()
pool.join()
else:
# If we have only one thread, skip the pool
# this is primarily to make debugging easier.
result = {
repository.name: worker(repository)[1]
for repository in all_repos
}
return result
def push_to_origin_if_not_upstream(self, repository, branch):
"""Push the local repository back to the origin, but not upstream."""
git_dir = repository.git_dir
origin = repository.origin
upstream = repository.upstream_or_none()
if upstream is None:
logging.warning('Skipping push origin %s because upstream is None.',
repository.name)
return
if origin == upstream:
logging.warning('Skipping push origin %s because origin is upstream.',
repository.name)
return
self.__git.push_branch_to_origin(git_dir, branch)
def determine_source_repositories(self):
"""Determine which repositories are available to this SCM."""
raise_and_log_error(
UnexpectedError(self.__class__.__name__
+ ': Should only be applicable to BomSCM',
cause='NotReachable'))
class PublishSpinnakerCommand(CommandProcessor):
""""Implements the publish_spinnaker command."""
# pylint: disable=too-few-public-methods
def __init__(self, factory, options, **kwargs):
super(PublishSpinnakerCommand, self).__init__(factory, options,
**kwargs)
check_options_set(options, [
'spinnaker_version', 'spinnaker_release_alias', 'bom_version',
'changelog_gist_url', 'github_owner', 'min_halyard_version'
])
major, minor, _ = self.options.spinnaker_version.split('.')
self.__branch = 'release-{major}.{minor}.x'.format(major=major,
minor=minor)
options_copy = copy.copy(options)
self.__scm = BomSourceCodeManager(options_copy, self.get_input_dir())
self.__hal = HalRunner(options)
self.__git = GitRunner(options)
self.__hal.check_property('spinnaker.config.input.bucket',
options.halyard_bom_bucket)
if options.only_repositories:
self.__only_repositories = options.only_repositories.split(',')
else:
self.__only_repositories = []
options_copy.git_branch = self.__branch
self.__process_scm = BranchSourceCodeManager(options_copy,
self.get_input_dir())
def push_branches_and_tags(self, bom):
"""Update the release branches and tags in each of the BOM repositires."""
logging.info('Tagging each of the BOM service repos')
# Run in two passes so we dont push anything if we hit a problem
# in the tagging pass. Since we are spread against multiple repositiories,
# we cannot do this atomically. The two passes gives us more protection
# from a partial push due to errors in a repo.
names_to_push = set([])
for which in ['tag', 'push']:
for name, spec in bom['services'].items():
if name in ['monitoring-third-party', 'defaultArtifact']:
# Ignore this, it is redundant to monitoring-daemon
continue
if name == 'monitoring-daemon':
name = 'spinnaker-monitoring'
if self.__only_repositories and name not in self.__only_repositories:
logging.debug('Skipping %s because of --only_repositories',
name)
continue
if spec is None:
logging.warning('HAVE bom.services.%s = None', name)
continue
repository = self.__scm.make_repository_spec(name)
self.__scm.ensure_local_repository(repository)
if which == 'tag':
added = self.__branch_and_tag_repository(
repository, self.__branch)
if added:
names_to_push.add(name)
else:
self.__push_branch_and_maybe_tag_repository(
repository, self.__branch, name in names_to_push)
for name in SPINNAKER_PROCESS_REPOSITORY_NAMES:
if self.__only_repositories and name not in self.__only_repositories:
logging.debug('Skipping %s because of --only_repositories',
name)
continue
repository = self.__process_scm.make_repository_spec(name)
self.__process_scm.ensure_local_repository(repository)
if self.__branch_and_tag_repository(repository, self.__branch):
self.__push_branch_and_maybe_tag_repository(
repository, self.__branch)
def __already_have_tag(self, repository, tag):
"""Determine if we already have the tag in the repository."""
git_dir = repository.git_dir
existing_commit = self.__git.query_commit_at_tag(git_dir, tag)
if not existing_commit:
return False
want_commit = self.__git.query_local_repository_commit_id(git_dir)
if want_commit == existing_commit:
logging.debug('Already have "%s" at %s', tag, want_commit)
return True
raise_and_log_error(
ConfigError(
'"{tag}" already exists in "{repo}" at commit {have}, not {want}'
.format(tag=tag,
repo=git_dir,
have=existing_commit,
want=want_commit)))
def __branch_and_tag_repository(self, repository, branch):
"""Create a branch and/or verison tag in the repository, if needed."""
version = self.__scm.determine_repository_version(repository)
tag = 'version-' + version
if self.__already_have_tag(repository, tag):
return False
self.__git.check_run(repository.git_dir, 'tag ' + tag)
return True
def __push_branch_and_maybe_tag_repository(self, repository, branch,
also_tag):
"""Push the branch and verison tag to the origin."""
tag = 'version-' + self.__scm.determine_repository_version(repository)
self.__git.push_branch_to_origin(repository.git_dir, branch)
if also_tag:
self.__git.push_tag_to_origin(repository.git_dir, tag)
else:
logging.info('%s was already tagged with "%s" -- skip',
repository.git_dir, tag)
def _do_command(self):
"""Implements CommandProcessor interface."""
options = self.options
spinnaker_version = options.spinnaker_version
options_copy = copy.copy(options)
options_copy.git_branch = 'master' # push to master in spinnaker.github.io
publish_changelog_command = PublishChangelogFactory().make_command(
options_copy)
changelog_gist_url = options.changelog_gist_url
# Make sure changelog exists already.
# If it does not then fail.
try:
logging.debug('Verifying changelog ready at %s',
changelog_gist_url)
urlopen(changelog_gist_url)
except HTTPError as error:
logging.error(exception_to_message)
raise_and_log_error(
ConfigError(
'Changelog gist "{url}" must exist before publising a release.'
.format(url=changelog_gist_url),
cause='ChangelogMissing'))
bom = self.__hal.retrieve_bom_version(self.options.bom_version)
bom['version'] = spinnaker_version
bom_path = os.path.join(self.get_output_dir(),
spinnaker_version + '.yml')
write_to_path(yaml.safe_dump(bom, default_flow_style=False), bom_path)
self.__hal.publish_bom_path(bom_path)
self.push_branches_and_tags(bom)
self.__hal.publish_spinnaker_release(spinnaker_version,
options.spinnaker_release_alias,
changelog_gist_url,
options.min_halyard_version)
logging.info('Publishing changelog')
publish_changelog_command()
class PushChangelogCommand(CommandProcessor):
"""Implements push_changelog_to_gist."""
def __init__(self, factory, options, **kwargs):
super(PushChangelogCommand, self).__init__(factory, options, **kwargs)
check_options_set(
options, ['build_changelog_gist_url', 'git_branch'])
if not options.changelog_path:
options.changelog_path = os.path.join(
self.get_output_dir(command=BUILD_CHANGELOG_COMMAND), 'changelog.md')
check_path_exists(options.changelog_path, why='changelog_path')
self.__git = GitRunner(options)
def _do_command(self):
options = self.options
gist_url = options.build_changelog_gist_url
index = gist_url.rfind('/')
if index < 0:
index = gist_url.rfind(':') # ssh gist
gist_id = gist_url[index + 1:]
git_dir = os.path.join(self.get_input_dir(), gist_id)
if not os.path.exists(git_dir):
logging.debug('Cloning gist from %s', gist_url)
ensure_dir_exists(os.path.dirname(git_dir))
self.__git.check_run(os.path.dirname(git_dir), 'clone ' + gist_url)
else:
logging.debug('Updating gist in "%s"', git_dir)
self.__git.check_run(git_dir, 'fetch origin master')
self.__git.check_run(git_dir, 'checkout master')
dest_path = os.path.join(
git_dir, '%s-raw-changelog.md' % options.git_branch)
logging.debug('Copying "%s" to "%s"', options.changelog_path, dest_path)
shutil.copyfile(options.changelog_path, dest_path)
self.__git.check_run(git_dir, 'add ' + os.path.basename(dest_path))
self.__git.check_commit_or_no_changes(
git_dir, '-a -m "Updated %s"' % os.path.basename(dest_path))
logging.debug('Pushing back gist')
self.__git.check_run(git_dir, 'push -f origin master')
def __init__(self, factory, options, **kwargs):
super(InitiateReleaseBranchCommand,
self).__init__(factory, options, **kwargs)
check_options_set(options, ['spinnaker_version'])
self.__git = GitRunner(options)
def setUpClass(cls):
cls.git = GitRunner(make_default_options())
cls.base_temp_dir = tempfile.mkdtemp(prefix='git_test')
cls.git_dir = os.path.join(cls.base_temp_dir, 'commit_message_test')
os.makedirs(cls.git_dir)
git_dir = cls.git_dir
gitify = lambda args: 'git -C "{dir}" {args}'.format(dir=git_dir,
args=args)
check_subprocess_sequence([
gitify('init'),
'touch "{dir}/base_file"'.format(dir=git_dir),
gitify('add "{dir}/base_file"'.format(dir=git_dir)),
gitify('commit -a -m "feat(test): added file"'),
gitify(
'tag {base_version} HEAD'.format(base_version=VERSION_BASE)),
# For testing patches
gitify('checkout -b {patch_branch}'.format(
patch_branch=cls.PATCH_BRANCH)),
'touch "{dir}/patch_file"'.format(dir=git_dir),
gitify('add "{dir}/patch_file"'.format(dir=git_dir)),
gitify('commit -a -m "fix(testA): added patch_file"'),
# For testing minor versions
gitify('checkout -b {minor_branch}'.format(
minor_branch=cls.MINOR_BRANCH)),
'touch "{dir}/minor_file"'.format(dir=git_dir),
gitify('add "{dir}/minor_file"'.format(dir=git_dir)),
gitify('commit -a -m "feat(testB): added minor_file"'),
# For testing major versions
gitify('checkout -b {major_branch}'.format(
major_branch=cls.MAJOR_BRANCH)),
'touch "{dir}/major_file"'.format(dir=git_dir),
gitify('add "{dir}/major_file"'.format(dir=git_dir)),
gitify('commit -a -m'
' "feat(testC): added major_file\n'
'\nInterestingly enough, this is a BREAKING CHANGE.'
'"'),
# For testing composite commits from a merge of commits
gitify('checkout -b {merged_branch}'.format(
merged_branch=cls.MERGED_BRANCH)),
gitify('reset --hard HEAD~3'),
gitify('merge --squash HEAD@{1}'),
])
env = dict(os.environ)
if os.path.exists('/bin/true'):
env['EDITOR'] = '/bin/true'
elif os.path.exists('/usr/bin/true'):
env['EDITOR'] = '/usr/bin/true'
else:
raise NotImplementedError('platform not supported for this test')
check_subprocess('git -C "{dir}" commit'.format(dir=git_dir), env=env)
# For testing changelog from a commit
check_subprocess_sequence([
gitify('checkout {minor_branch}'.format(
minor_branch=cls.MINOR_BRANCH)),
gitify('checkout -b {x_branch}'.format(
x_branch=cls.PATCH_MINOR_BRANCH)),
'touch "{dir}/xbefore_file"'.format(dir=git_dir),
gitify('add "{dir}/xbefore_file"'.format(dir=git_dir)),
gitify('commit -a -m "feat(test): COMMIT AT TAG"'),
gitify('tag {x_marker} HEAD'.format(x_marker=cls.PATCH_MINOR_X)),
'touch "{dir}/x_first"'.format(dir=git_dir),
gitify('add "{dir}/x_first"'.format(dir=git_dir)),
gitify('commit -a -m "fix(test): First Fix"'),
'rm "{dir}/x_first"'.format(dir=git_dir),
gitify('commit -a -m "fix(test): Second Fix"'),
])
class GradleRunner(object):
"""Helper module for running gradle."""
__GRADLE_PUBLISH_FILE = os.path.join("gradle", "init-publish.gradle")
@staticmethod
def add_parser_args(parser, defaults):
"""Add parser arguments for gradle."""
if hasattr(parser, "added_gradle_runner"):
return
parser.added_gradle_runner = True
add_parser_argument(
parser,
"bintray_jar_repository",
defaults,
None,
help=
"bintray repository in the bintray_org to publish jar files into.",
)
add_parser_argument(
parser,
"gradle_cache_path",
defaults,
"{home}/.gradle".format(
home=os.environ["HOME"]) if os.environ.get("HOME") else None,
help="Path to a gradle cache directory to use for the builds.",
)
add_parser_argument(
parser,
"gradle_network_timeout_secs",
defaults,
60,
help="Seconds to configure gradle timeouts (e.g. with bintray).",
)
add_parser_argument(
parser,
"maven_custom_init_file",
defaults,
os.path.join(os.path.dirname(__file__), "..", "maven-init.gradle"),
help="Path to a gradle init file to add to the debian builds."
" Used to specify any custom behavior in the gradle builds."
" Argument is a file path relative to the directory this script is"
" executed in."
" The default value assumes we run this script from the parent"
" directory of spinnaker/buildtool.",
)
@property
def source_code_manager(self):
"""Return bound source code manager."""
return self.__scm
def __init__(self, options, scm, metrics):
self.__options = options
self.__metrics = metrics
self.__git = GitRunner(options)
self.__scm = scm
def __to_bintray_url(self, repo, package_name, repository, build_version):
"""Return the url for the desired versioned repository in bintray repo."""
bintray_path = "packages/{subject}/{repo}/{package}/versions/{version}".format(
subject=self.__options.bintray_org,
package=package_name,
repo=repo,
version=build_version,
)
return "https://api.bintray.com/" + bintray_path
def __add_bintray_auth_header(self, request):
"""Adds bintray authentication header to the request."""
user = os.environ["BINTRAY_USER"]
password = os.environ["BINTRAY_KEY"]
encoded_auth = base64.b64encode("{user}:{password}".format(
user=user, password=password))
request.add_header("Authorization",
"Basic " + bytes.decode(encoded_auth))
def bintray_repo_has_version(self, repo, package_name, repository,
build_version):
"""See if the given bintray repository has the package version to build."""
try:
bintray_url = self.__to_bintray_url(repo, package_name, repository,
build_version)
logging.debug("Checking for %s", bintray_url)
request = Request(url=bintray_url)
self.__add_bintray_auth_header(request)
urlopen(request)
return True
except HTTPError as ex:
if ex.code == 404:
return False
raise_and_log_error(
ResponseError("Bintray failure: {}".format(ex),
server="bintray.check"),
"Failed on url=%s: %s" %
(bintray_url, exception_to_message(ex)),
)
except Exception as ex:
raise
def consider_debian_on_bintray(self, repository, build_version):
"""Check whether desired version already exists on bintray."""
options = self.__options
exists = []
missing = []
# technically we publish to both maven and debian repos.
# we can be in a state where we are in one but not the other.
# let's not worry about this for now.
for bintray_repo in [options.bintray_debian_repository]: # ,
# options.bintray_jar_repository]:
package_name = repository.name
if bintray_repo == options.bintray_debian_repository:
if package_name == "spinnaker-monitoring":
package_name = "spinnaker-monitoring-daemon"
elif not package_name.startswith("spinnaker"):
package_name = "spinnaker-" + package_name
if self.bintray_repo_has_version(bintray_repo, package_name,
repository, build_version):
exists.append(bintray_repo)
else:
missing.append(bintray_repo)
if exists:
if options.skip_existing:
if missing:
raise_and_log_error(
ConfigError(
"Have {name} version for {exists} but not {missing}"
.format(
name=repository.name,
exists=exists[0],
missing=missing[0],
)))
logging.info("Already have %s -- skipping build",
repository.name)
labels = {"repository": repository.name, "artifact": "debian"}
self.__metrics.inc_counter("ReuseArtifact", labels)
return True
if options.delete_existing:
for repo in exists:
self.bintray_repo_delete_version(
repo,
package_name,
repository,
build_version=build_version)
else:
raise_and_log_error(
ConfigError("Already have debian for {name}".format(
name=repository.name)))
return False
def bintray_repo_delete_version(self,
repo,
package_name,
repository,
build_version=None):
"""Delete the given bintray repository version if it exsts."""
try:
bintray_url = self.__to_bintray_url(repo, package_name, repository,
build_version)
logging.debug("Checking for %s", bintray_url)
request = Request(url=bintray_url)
request.get_method = lambda: "DELETE"
self.__add_bintray_auth_header(request)
labels = {
"repo": repo,
"repository": repository.name,
"artifact": "debian"
}
self.__metrics.count_call("DeleteArtifact", labels, urlopen,
request)
return True
except HTTPError as ex:
if ex.code == 404:
return True
raise_and_log_error(
ResponseError("Bintray failure: {}".format(ex),
server="bintray.delete"),
"Failed on url=%s: %s" %
(bintray_url, exception_to_message(ex)),
)
def get_common_args(self):
"""Return standard gradle args."""
options = self.__options
args = [
"--stacktrace",
"--info",
]
if options.maven_custom_init_file:
# Note, this was only debians
args.append("-I {}".format(options.maven_custom_init_file))
return args
def get_debian_args(self, distribution):
"""Return the debian args for the given distribution name."""
bintray_key = os.environ["BINTRAY_KEY"]
bintray_user = os.environ["BINTRAY_USER"]
options = self.__options
bintray_org = options.bintray_org
jar_repo = options.bintray_jar_repository
debian_repo = options.bintray_debian_repository
publish_wait_secs = options.bintray_publish_wait_secs
args = [
'-PbintrayOrg="{org}"'.format(org=bintray_org),
'-PbintrayPackageRepo="{repo}"'.format(repo=debian_repo),
'-PbintrayJarRepo="{jarRepo}"'.format(jarRepo=jar_repo),
'-PbintrayKey="{key}"'.format(key=bintray_key),
'-PbintrayUser="******"'.format(user=bintray_user),
"-PbintrayPackageDebDistribution={distribution}".format(
distribution=distribution),
"-PbintrayPublishWaitForSecs={publishWaitSecs}".format(
publishWaitSecs=publish_wait_secs),
]
return args
def check_run(
self,
args,
command_processor,
repository,
target,
context,
version,
build_number,
gradle_dir=None,
):
"""Run the gradle command on the given repository."""
gradle_dir = gradle_dir or repository.git_dir
if self.__has_init_publish_file(repository):
args.extend(["-I", self.__GRADLE_PUBLISH_FILE])
if self.__is_plugin_version_6(repository):
args.extend(["-Pversion=%s-%s" % (version, build_number)])
else:
args.append("-Prelease.useLastTag=true")
build_number = self.prepare_local_git_for_nebula(
gradle_dir,
repository,
version=version,
build_number=build_number)
full_args = list(args)
full_args.append("-PbintrayPackageBuildNumber=%s" % build_number)
# This gradle options wasnt introduced until 4.10.2
timeout = self.__options.gradle_network_timeout_secs * 1000
if timeout:
full_args.append("-Dorg.gradle.internal.http.socketTimeout=%d" %
timeout)
full_args.append(
"-Dorg.gradle.internal.http.connectionTimeout=%d" % timeout)
name = repository.name
logfile = command_processor.get_logfile_path(name + "-" + context)
cmd = "./gradlew {args} {target}".format(args=" ".join(full_args),
target=target)
labels = {
"repository": repository.name,
"context": context,
"target": target
}
self.__metrics.time_call(
"GradleBuild",
labels,
self.__metrics.default_determine_outcome_labels,
check_subprocesses_to_logfile,
name + " gradle " + context,
logfile,
[cmd],
cwd=gradle_dir,
postprocess_hook=GradleMetricsUpdater(self.__metrics, repository,
target),
)
def __is_plugin_version_6(self, repository):
return (not self.__has_init_publish_file(repository)
and not repository.name == "spinnaker-monitoring")
def __has_init_publish_file(self, repository):
return os.path.isfile(
os.path.join(repository.git_dir, self.__GRADLE_PUBLISH_FILE))
def prepare_local_git_for_nebula(self,
gradle_dir,
repository,
version=None,
build_number=None):
"""Tag the repository with the version we want to build.
Args:
version: optional version to tag with. If not provided then infer it.
gradle_dir: The dir to prepare if supplied
"""
git_dir = gradle_dir or repository.git_dir
self.__scm.ensure_local_repository(repository)
self.__git.remove_all_non_version_tags(repository, git_dir=git_dir)
if not build_number:
build_number = self.__scm.determine_build_number(repository)
# This doesn't really work because get_repository_service_build_version only
# exists in BomSourceCodeManager.
if not version:
build_version = self.__scm.get_repository_service_build_version(
repository)
else:
build_version = "%s-%s" % (version, build_number)
logging.debug('Tagging repository %s with "%s" for nebula', git_dir,
build_version)
self.__git.tag_head(git_dir, build_version)
return build_number
class GradleRunner(object):
"""Helper module for running gradle."""
@staticmethod
def add_parser_args(parser, defaults):
"""Add parser arguments for gradle."""
if hasattr(parser, 'added_gradle_runner'):
return
parser.added_gradle_runner = True
add_parser_argument(
parser,
'bintray_jar_repository',
defaults,
None,
help=
'bintray repository in the bintray_org to publish jar files into.')
add_parser_argument(
parser,
'gradle_cache_path',
defaults,
'{home}/.gradle'.format(
home=os.environ['HOME']) if os.environ.get('HOME') else None,
help='Path to a gradle cache directory to use for the builds.')
add_parser_argument(
parser,
'maven_custom_init_file',
defaults,
os.path.join(os.path.dirname(__file__), '..', 'maven-init.gradle'),
help='Path to a gradle init file to add to the debian builds.'
' Used to specify any custom behavior in the gradle builds.'
' Argument is a file path relative to the directory this script is'
' executed in.'
' The default value assumes we run this script from the parent'
' directory of spinnaker/spinnaker.')
@property
def source_code_manager(self):
"""Return bound source code manager."""
return self.__scm
def __init__(self, options, scm, metrics):
self.__options = options
self.__metrics = metrics
self.__git = GitRunner(options)
self.__scm = scm
def __to_bintray_url(self,
repo,
package_name,
repository,
build_version=None,
build_number=None):
"""Return the url for the desired versioned repository in bintray repo."""
if not build_version:
source_info = self.__scm.lookup_source_info(repository)
build_number = build_number or source_info.build_number
build_version = '%s-%s' % (source_info.summary.version,
build_number)
bintray_path = (
'packages/{subject}/{repo}/{package}/versions/{version}'.format(
subject=self.__options.bintray_org,
package=package_name,
repo=repo,
version=build_version))
return 'https://api.bintray.com/' + bintray_path
def __add_bintray_auth_header(self, request):
"""Adds bintray authentication header to the request."""
user = os.environ['BINTRAY_USER']
password = os.environ['BINTRAY_KEY']
encoded_auth = base64.encodestring('{user}:{password}'.format(
user=user, password=password))[:-1] # strip eoln
request.add_header('Authorization', 'Basic ' + encoded_auth)
def bintray_repo_has_version(self,
repo,
package_name,
repository,
build_version=None,
build_number=None):
"""See if the given bintray repository has the package version to build."""
try:
bintray_url = self.__to_bintray_url(repo,
package_name,
repository,
build_version=build_version,
build_number=build_number)
logging.debug('Checking for %s', bintray_url)
request = urllib2.Request(url=bintray_url)
self.__add_bintray_auth_header(request)
urllib2.urlopen(request)
return True
except urllib2.HTTPError as ex:
if ex.code == 404:
return False
raise_and_log_error(
ResponseError('Bintray failure: {}'.format(ex),
server='bintray.check'),
'Failed on url=%s: %s' % (bintray_url, ex.message))
except Exception as ex:
raise
def consider_debian_on_bintray(self,
repository,
build_version=None,
build_number=None):
"""Check whether desired version already exists on bintray."""
options = self.__options
exists = []
missing = []
# technically we publish to both maven and debian repos.
# we can be in a state where we are in one but not the other.
# let's not worry about this for now.
for bintray_repo in [options.bintray_debian_repository]: #,
# options.bintray_jar_repository]:
package_name = repository.name
if bintray_repo == options.bintray_debian_repository:
if package_name == 'spinnaker-monitoring':
package_name = 'spinnaker-monitoring-daemon'
elif not package_name.startswith('spinnaker'):
package_name = 'spinnaker-' + package_name
if self.bintray_repo_has_version(bintray_repo,
package_name,
repository,
build_version=build_version,
build_number=build_number):
exists.append(bintray_repo)
else:
missing.append(bintray_repo)
if exists:
if options.skip_existing:
if missing:
raise_and_log_error(
ConfigError(
'Have {name} version for {exists} but not {missing}'
.format(name=repository.name,
exists=exists[0],
missing=missing[0])))
logging.info('Already have %s -- skipping build',
repository.name)
labels = {'repository': repository.name, 'artifact': 'debian'}
self.__metrics.inc_counter(
'ReuseArtifact', labels,
'Kept existing desired debian package version.')
return True
if options.delete_existing:
for repo in exists:
self.bintray_repo_delete_version(
repo,
package_name,
repository,
build_version=build_version)
else:
raise_and_log_error(
ConfigError('Already have debian for {name}'.format(
name=repository.name)))
return False
def bintray_repo_delete_version(self,
repo,
package_name,
repository,
build_version=None):
"""Delete the given bintray repository version if it exsts."""
try:
bintray_url = self.__to_bintray_url(repo,
package_name,
repository,
build_version=build_version)
logging.debug('Checking for %s', bintray_url)
request = urllib2.Request(url=bintray_url)
request.get_method = lambda: 'DELETE'
self.__add_bintray_auth_header(request)
labels = {
'repo': repo,
'repository': repository.name,
'artifact': 'debian'
}
self.__metrics.count_call(
'DeleteArtifact', labels,
'Attempts to delete versioned artifacts on bintray',
urllib2.urlopen, request)
return True
except urllib2.HTTPError as ex:
if ex.code == 404:
return True
raise_and_log_error(
ResponseError('Bintray failure: {}'.format(ex),
server='bintray.delete'),
'Failed on url=%s: %s' % (bintray_url, ex.message))
def get_common_args(self):
"""Return standard gradle args."""
options = self.__options
args = [
'--stacktrace',
'--info',
'-Prelease.useLastTag=true',
]
if options.maven_custom_init_file:
# Note, this was only debians, not for rpms before.
args.append('-I {}'.format(options.maven_custom_init_file))
return args
def get_debian_args(self, distribution):
"""Return the debian args for the given distribution name."""
bintray_key = os.environ['BINTRAY_KEY']
bintray_user = os.environ['BINTRAY_USER']
options = self.__options
bintray_org = options.bintray_org
jar_repo = options.bintray_jar_repository
debian_repo = options.bintray_debian_repository
args = [
'-PbintrayOrg="{org}"'.format(org=bintray_org),
'-PbintrayPackageRepo="{repo}"'.format(repo=debian_repo),
'-PbintrayJarRepo="{jarRepo}"'.format(jarRepo=jar_repo),
'-PbintrayKey="{key}"'.format(key=bintray_key),
'-PbintrayUser="******"'.format(user=bintray_user),
'-PbintrayPackageDebDistribution={distribution}'.format(
distribution=distribution)
]
return args
def check_run(self,
args,
command_processor,
repository,
target,
context,
gradle_dir=None,
**kwargs):
"""Run the gradle command on the given repository."""
gradle_dir = gradle_dir or repository.git_dir
version = kwargs.pop('version', None)
build_number = kwargs.pop('build_number', None)
build_number = self.prepare_local_git_for_nebula(
gradle_dir, repository, version=version, build_number=build_number)
full_args = list(args)
full_args.append('-PbintrayPackageBuildNumber=%s' % build_number)
name = repository.name
logfile = command_processor.get_logfile_path(name + '-' + context)
cmd = './gradlew {args} {target}'.format(args=' '.join(full_args),
target=target)
labels = {
'repository': repository.name,
'context': context,
'target': target
}
self.__metrics.time_call('GradleBuild',
labels,
'Gradle builds.',
check_subprocesses_to_logfile,
name + ' gradle ' + context,
logfile, [cmd],
cwd=gradle_dir,
postprocess_hook=GradleMetricsUpdater(
self.__metrics, repository, target))
def prepare_local_git_for_nebula(self,
gradle_dir,
repository,
version=None,
build_number=None):
"""Tag the repository with the version we want to build.
Args:
version: optional version to tag with. If not provided then infer it.
gradle_dir: The dir to prepare if supplied
"""
git_dir = gradle_dir or repository.git_dir
self.__scm.ensure_local_repository(repository)
self.__git.remove_all_non_version_tags(repository, git_dir=git_dir)
if not version or not build_number:
source_info = self.__scm.lookup_source_info(repository)
if not build_number:
build_number = source_info.build_number
if not version:
version = source_info.summary.version
build_version = '%s-%s' % (version, build_number)
logging.debug('Tagging repository %s with "%s" for nebula', git_dir,
build_version)
self.__git.tag_head(git_dir, build_version)
return build_number
def __init__(self, options, scm, metrics):
self.__options = options
self.__metrics = metrics
self.__git = GitRunner(options)
self.__scm = scm
def __init__(self, options, scm, metrics): self.__options = options self.__metrics = metrics self.__git = GitRunner(options) self.__scm = scm
class GradleRunner(object):
"""Helper module for running gradle."""
__GRADLE_PUBLISH_FILE = os.path.join("gradle", "init-publish.gradle")
@staticmethod
def add_parser_args(parser, defaults):
"""Add parser arguments for gradle."""
if hasattr(parser, 'added_gradle_runner'):
return
parser.added_gradle_runner = True
add_parser_argument(
parser, 'bintray_jar_repository', defaults, None,
help='bintray repository in the bintray_org to publish jar files into.')
add_parser_argument(
parser, 'gradle_cache_path', defaults,
'{home}/.gradle'.format(home=os.environ['HOME'])
if os.environ.get('HOME') else None,
help='Path to a gradle cache directory to use for the builds.')
add_parser_argument(
parser, 'gradle_network_timeout_secs', defaults, 60,
help='Seconds to configure gradle timeouts (e.g. with bintray).')
add_parser_argument(
parser, 'maven_custom_init_file', defaults,
os.path.join(os.path.dirname(__file__), '..', 'maven-init.gradle'),
help='Path to a gradle init file to add to the debian builds.'
' Used to specify any custom behavior in the gradle builds.'
' Argument is a file path relative to the directory this script is'
' executed in.'
' The default value assumes we run this script from the parent'
' directory of spinnaker/spinnaker.')
@property
def source_code_manager(self):
"""Return bound source code manager."""
return self.__scm
def __init__(self, options, scm, metrics):
self.__options = options
self.__metrics = metrics
self.__git = GitRunner(options)
self.__scm = scm
def __to_bintray_url(self, repo, package_name, repository, build_version):
"""Return the url for the desired versioned repository in bintray repo."""
bintray_path = (
'packages/{subject}/{repo}/{package}/versions/{version}'.format(
subject=self.__options.bintray_org,
package=package_name, repo=repo, version=build_version))
return 'https://api.bintray.com/' + bintray_path
def __add_bintray_auth_header(self, request):
"""Adds bintray authentication header to the request."""
user = os.environ['BINTRAY_USER']
password = os.environ['BINTRAY_KEY']
encoded_auth = base64.encodestring(str.encode('{user}:{password}'.format(
user=user, password=password)))[:-1] # strip eoln
request.add_header('Authorization', 'Basic ' + bytes.decode(encoded_auth))
def bintray_repo_has_version(self, repo, package_name, repository,
build_version):
"""See if the given bintray repository has the package version to build."""
try:
bintray_url = self.__to_bintray_url(repo, package_name, repository,
build_version)
logging.debug('Checking for %s', bintray_url)
request = Request(url=bintray_url)
self.__add_bintray_auth_header(request)
urlopen(request)
return True
except HTTPError as ex:
if ex.code == 404:
return False
raise_and_log_error(
ResponseError('Bintray failure: {}'.format(ex),
server='bintray.check'),
'Failed on url=%s: %s' % (bintray_url, exception_to_message(ex)))
except Exception as ex:
raise
def consider_debian_on_bintray(self, repository, build_version):
"""Check whether desired version already exists on bintray."""
options = self.__options
exists = []
missing = []
# technically we publish to both maven and debian repos.
# we can be in a state where we are in one but not the other.
# let's not worry about this for now.
for bintray_repo in [options.bintray_debian_repository]:#,
# options.bintray_jar_repository]:
package_name = repository.name
if bintray_repo == options.bintray_debian_repository:
if package_name == 'spinnaker-monitoring':
package_name = 'spinnaker-monitoring-daemon'
elif not package_name.startswith('spinnaker'):
package_name = 'spinnaker-' + package_name
if self.bintray_repo_has_version(
bintray_repo, package_name, repository, build_version):
exists.append(bintray_repo)
else:
missing.append(bintray_repo)
if exists:
if options.skip_existing:
if missing:
raise_and_log_error(
ConfigError('Have {name} version for {exists} but not {missing}'
.format(name=repository.name,
exists=exists[0], missing=missing[0])))
logging.info('Already have %s -- skipping build', repository.name)
labels = {'repository': repository.name, 'artifact': 'debian'}
self.__metrics.inc_counter('ReuseArtifact', labels)
return True
if options.delete_existing:
for repo in exists:
self.bintray_repo_delete_version(repo, package_name, repository,
build_version=build_version)
else:
raise_and_log_error(
ConfigError('Already have debian for {name}'.format(
name=repository.name)))
return False
def bintray_repo_delete_version(self, repo, package_name, repository,
build_version=None):
"""Delete the given bintray repository version if it exsts."""
try:
bintray_url = self.__to_bintray_url(repo, package_name, repository,
build_version)
logging.debug('Checking for %s', bintray_url)
request = Request(url=bintray_url)
request.get_method = lambda: 'DELETE'
self.__add_bintray_auth_header(request)
labels = {
'repo': repo,
'repository': repository.name,
'artifact': 'debian'
}
self.__metrics.count_call(
'DeleteArtifact', labels, urlopen, request)
return True
except HTTPError as ex:
if ex.code == 404:
return True
raise_and_log_error(
ResponseError('Bintray failure: {}'.format(ex),
server='bintray.delete'),
'Failed on url=%s: %s' % (bintray_url, exception_to_message(ex)))
def get_common_args(self):
"""Return standard gradle args."""
options = self.__options
args = [
'--stacktrace',
'--info',
]
if options.maven_custom_init_file:
# Note, this was only debians, not for rpms before.
args.append('-I {}'.format(options.maven_custom_init_file))
return args
def get_debian_args(self, distribution):
"""Return the debian args for the given distribution name."""
bintray_key = os.environ['BINTRAY_KEY']
bintray_user = os.environ['BINTRAY_USER']
options = self.__options
bintray_org = options.bintray_org
jar_repo = options.bintray_jar_repository
debian_repo = options.bintray_debian_repository
publish_wait_secs = options.bintray_publish_wait_secs
args = [
'-PbintrayOrg="{org}"'.format(org=bintray_org),
'-PbintrayPackageRepo="{repo}"'.format(repo=debian_repo),
'-PbintrayJarRepo="{jarRepo}"'.format(jarRepo=jar_repo),
'-PbintrayKey="{key}"'.format(key=bintray_key),
'-PbintrayUser="******"'.format(user=bintray_user),
'-PbintrayPackageDebDistribution={distribution}'.format(
distribution=distribution),
'-PbintrayPublishWaitForSecs={publishWaitSecs}'.format(
publishWaitSecs=publish_wait_secs)
]
return args
def check_run(self, args, command_processor, repository, target, context,
version, build_number, gradle_dir=None):
"""Run the gradle command on the given repository."""
gradle_dir = gradle_dir or repository.git_dir
if self.__has_init_publish_file(repository):
args.extend(['-I', self.__GRADLE_PUBLISH_FILE])
if self.__is_plugin_version_6(repository):
args.extend(['-PenablePublishing=true',
'-Prelease.disableGitChecks=true',
'-Prelease.version=%s-%s' % (version, build_number)])
else:
args.append('-Prelease.useLastTag=true')
build_number = self.prepare_local_git_for_nebula(
gradle_dir, repository, version=version, build_number=build_number)
full_args = list(args)
full_args.append('-PbintrayPackageBuildNumber=%s' % build_number)
# This gradle options wasnt introduced until 4.10.2
timeout = self.__options.gradle_network_timeout_secs * 1000
if timeout:
full_args.append('-Dorg.gradle.internal.http.socketTimeout=%d' % timeout)
full_args.append('-Dorg.gradle.internal.http.connectionTimeout=%d' % timeout)
name = repository.name
logfile = command_processor.get_logfile_path(name + '-' + context)
cmd = './gradlew {args} {target}'.format(
args=' '.join(full_args), target=target)
labels = {
'repository': repository.name,
'context': context,
'target': target
}
self.__metrics.time_call(
'GradleBuild', labels, self.__metrics.default_determine_outcome_labels,
check_subprocesses_to_logfile,
name + ' gradle ' + context, logfile, [cmd], cwd=gradle_dir,
postprocess_hook=GradleMetricsUpdater(self.__metrics,
repository, target))
def __is_plugin_version_6(self, repository):
return not self.__has_init_publish_file(
repository) and not repository.name == 'spinnaker-monitoring'
def __has_init_publish_file(self, repository):
return os.path.isfile(
os.path.join(repository.git_dir, self.__GRADLE_PUBLISH_FILE))
def prepare_local_git_for_nebula(
self, gradle_dir, repository, version=None, build_number=None):
"""Tag the repository with the version we want to build.
Args:
version: optional version to tag with. If not provided then infer it.
gradle_dir: The dir to prepare if supplied
"""
git_dir = gradle_dir or repository.git_dir
self.__scm.ensure_local_repository(repository)
self.__git.remove_all_non_version_tags(repository, git_dir=git_dir)
if not build_number:
build_number = self.__scm.determine_build_number(repository)
# This doesn't really work because get_repository_service_build_version only
# exists in BomSourceCodeManager.
if not version:
build_version = self.__scm.get_repository_service_build_version(
repository)
else:
build_version = '%s-%s' % (version, build_number)
logging.debug('Tagging repository %s with "%s" for nebula',
git_dir, build_version)
self.__git.tag_head(git_dir, build_version)
return build_number
class PublishSpinnakerCommand(CommandProcessor):
""""Implements the publish_spinnaker command."""
# pylint: disable=too-few-public-methods
def __init__(self, factory, options, **kwargs):
super(PublishSpinnakerCommand, self).__init__(factory, options, **kwargs)
check_options_set(options, [
'spinnaker_version',
'spinnaker_release_alias',
'bom_version',
'changelog_gist_url',
'github_owner',
'min_halyard_version'
])
major, minor, _ = self.options.spinnaker_version.split('.')
self.__branch = 'release-{major}.{minor}.x'.format(
major=major, minor=minor)
options_copy = copy.copy(options)
self.__bom_scm = BomSourceCodeManager(options_copy, self.get_input_dir())
self.__hal = HalRunner(options)
self.__git = GitRunner(options)
self.__hal.check_property(
'spinnaker.config.input.bucket', options.halyard_bom_bucket)
if options.only_repositories:
self.__only_repositories = options.only_repositories.split(',')
else:
self.__only_repositories = []
options_copy.git_branch = self.__branch
self.__branch_scm = BranchSourceCodeManager(
options_copy, self.get_input_dir())
def push_branches_and_tags(self, bom):
"""Update the release branches and tags in each of the BOM repositires."""
logging.info('Tagging each of the BOM service repos')
bom_scm = self.__bom_scm
branch_scm = self.__branch_scm
# Run in two passes so we dont push anything if we hit a problem
# in the tagging pass. Since we are spread against multiple repositiories,
# we cannot do this atomically. The two passes gives us more protection
# from a partial push due to errors in a repo.
names_to_push = set([])
for which in ['tag', 'push']:
for name, spec in bom['services'].items():
if name in ['monitoring-third-party', 'defaultArtifact']:
# Ignore this, it is redundant to monitoring-daemon
continue
if name == 'monitoring-daemon':
name = 'spinnaker-monitoring'
if self.__only_repositories and name not in self.__only_repositories:
logging.debug('Skipping %s because of --only_repositories', name)
continue
if spec is None:
logging.warning('HAVE bom.services.%s = None', name)
continue
repository = bom_scm.make_repository_spec(name)
bom_scm.ensure_local_repository(repository)
version = bom_scm.determine_repository_version(repository)
if which == 'tag':
added = self.__branch_and_tag_repository(
repository, self.__branch, version)
if added:
names_to_push.add(name)
else:
self.__push_branch_and_maybe_tag_repository(
repository, self.__branch, version, name in names_to_push)
additional_repositories = list(SPINNAKER_PROCESS_REPOSITORY_NAMES)
for name in additional_repositories:
if self.__only_repositories and name not in self.__only_repositories:
logging.debug('Skipping %s because of --only_repositories', name)
continue
repository = branch_scm.make_repository_spec(name)
branch_scm.ensure_local_repository(repository)
git_summary = self.__git.collect_repository_summary(repository.git_dir)
version = git_summary.version
if self.__branch_and_tag_repository(
repository, self.__branch, version):
self.__push_branch_and_maybe_tag_repository(
repository, self.__branch, version, True)
def __already_have_tag(self, repository, tag):
"""Determine if we already have the tag in the repository."""
git_dir = repository.git_dir
existing_commit = self.__git.query_commit_at_tag(git_dir, tag)
if not existing_commit:
return False
want_commit = self.__git.query_local_repository_commit_id(git_dir)
if want_commit == existing_commit:
logging.debug('Already have "%s" at %s', tag, want_commit)
return True
raise_and_log_error(
ConfigError(
'"{tag}" already exists in "{repo}" at commit {have}, not {want}'
.format(tag=tag, repo=git_dir,
have=existing_commit, want=want_commit)))
return False # not reached
def __branch_and_tag_repository(self, repository, branch, version):
"""Create a branch and/or version tag in the repository, if needed."""
tag = 'version-' + version
if self.__already_have_tag(repository, tag):
return False
self.__git.check_run(repository.git_dir, 'tag ' + tag)
return True
def __push_branch_and_maybe_tag_repository(self, repository, branch, version,
also_tag):
"""Push the branch and version tag to the origin."""
tag = 'version-' + version
self.__git.push_branch_to_origin(repository.git_dir, branch)
if also_tag:
self.__git.push_tag_to_origin(repository.git_dir, tag)
else:
logging.info('%s was already tagged with "%s" -- skip',
repository.git_dir, tag)
def _do_command(self):
"""Implements CommandProcessor interface."""
options = self.options
spinnaker_version = options.spinnaker_version
options_copy = copy.copy(options)
options_copy.git_branch = 'master' # push to master in spinnaker.github.io
publish_changelog_command = PublishChangelogFactory().make_command(
options_copy)
changelog_gist_url = options.changelog_gist_url
# Make sure changelog exists already.
# If it does not then fail.
try:
logging.debug('Verifying changelog ready at %s', changelog_gist_url)
urlopen(changelog_gist_url)
except HTTPError:
logging.error(exception_to_message)
raise_and_log_error(
ConfigError(
'Changelog gist "{url}" must exist before publising a release.'
.format(url=changelog_gist_url),
cause='ChangelogMissing'))
bom = self.__hal.retrieve_bom_version(self.options.bom_version)
bom['version'] = spinnaker_version
bom_path = os.path.join(self.get_output_dir(), spinnaker_version + '.yml')
write_to_path(yaml.safe_dump(bom, default_flow_style=False), bom_path)
self.__hal.publish_bom_path(bom_path)
self.push_branches_and_tags(bom)
self.__hal.publish_spinnaker_release(
spinnaker_version, options.spinnaker_release_alias, changelog_gist_url,
options.min_halyard_version)
logging.info('Publishing changelog')
publish_changelog_command()
def init_argparser(self, parser, defaults):
super(PublishHalyardCommandFactory,
self).init_argparser(parser, defaults)
GradleCommandFactory.add_bom_parser_args(parser, defaults)
SpinnakerSourceCodeManager.add_parser_args(parser, defaults)
GradleRunner.add_parser_args(parser, defaults)
GitRunner.add_publishing_parser_args(parser, defaults)
HalRunner.add_parser_args(parser, defaults)
self.add_argument(
parser,
'build_number',
defaults,
DEFAULT_BUILD_NUMBER,
help=
'Publishing halyard requires a rebuild. This is the build number'
' to use when rebuilding halyard.')
self.add_argument(
parser,
'halyard_version',
defaults,
None,
required=True,
help='The semantic version of the release to publish.')
self.add_argument(
parser,
'halyard_version_commits_url',
defaults,
None,
help='URL to file containing version and git commit for successful'
' nightly builds. By default this will be'
' "{filename}" in the'
' --halyard_bucket_base_url.'.format(
filename=BuildHalyardCommand.HALYARD_VERSIONS_BASENAME))
self.add_argument(
parser,
'halyard_bucket_base_url',
defaults,
None,
help='Base Google Cloud Storage URL for writing halyard builds.')
self.add_argument(parser,
'docs_repo_owner',
defaults,
None,
help='Owner of the docs repo if one was'
' specified. The default is --github_owner.')
self.add_argument(
parser,
'skip_existing',
defaults,
False,
type=bool,
help='Skip builds if the desired version already exists on bintray.'
)
self.add_argument(
parser,
'delete_existing',
defaults,
None,
type=bool,
help='Delete pre-existing desired versions if from bintray.')
class PushChangelogCommand(CommandProcessor):
"""Implements push_changelog_to_gist."""
def __init__(self, factory, options, **kwargs):
super(PushChangelogCommand, self).__init__(factory, options, **kwargs)
check_options_set(options, ['build_changelog_gist_url', 'git_branch'])
if not options.changelog_path:
options.changelog_path = os.path.join(
self.get_output_dir(command=BUILD_CHANGELOG_COMMAND),
'changelog.md')
check_path_exists(options.changelog_path, why='changelog_path')
self.__git = GitRunner(options)
def _do_command(self):
options = self.options
gist_url = options.build_changelog_gist_url
index = gist_url.rfind('/')
if index < 0:
index = gist_url.rfind(':') # ssh gist
gist_id = gist_url[index + 1:]
git_dir = os.path.join(self.get_input_dir(), gist_id)
if not os.path.exists(git_dir):
logging.debug('Cloning gist from %s', gist_url)
ensure_dir_exists(os.path.dirname(git_dir))
self.__git.check_run(os.path.dirname(git_dir), 'clone ' + gist_url)
else:
logging.debug('Updating gist in "%s"', git_dir)
self.__git.check_run(git_dir, 'fetch origin master')
self.__git.check_run(git_dir, 'checkout master')
dest_path = os.path.join(git_dir,
'%s-raw-changelog.md' % options.git_branch)
logging.debug('Copying "%s" to "%s"', options.changelog_path,
dest_path)
shutil.copyfile(options.changelog_path, dest_path)
self.__git.check_run(git_dir, 'add ' + os.path.basename(dest_path))
self.__git.check_commit_or_no_changes(
git_dir, '-a -m "Updated %s"' % os.path.basename(dest_path))
logging.debug('Pushing back gist')
self.__git.check_run(git_dir, 'push -f origin master')
def make_test_options(self):
options = super(TestBomSourceCodeManager, self).make_test_options()
parser = argparse.ArgumentParser()
parser.add_argument('--output_dir', default=options.output_dir)
GitRunner.add_parser_args(parser, {'github_owner': 'test_github_owner'})
return parser.parse_args()
