class PublishBomCommand(RepositoryCommandProcessor):
"""Implements publish_bom"""
def __init__(self, factory, options, **kwargs):
options.github_disable_upstream_push = True
super(PublishBomCommand, self).__init__(factory, options, **kwargs)
self.__hal_runner = HalRunner(options)
logging.debug('Verifying halyard server is consistent')
# Halyard is configured with fixed endpoints, however when we
# pubish we want to be explicit about where we are publishing to.
# There isnt a way to control this in halyard on a per-request basis
# so make sure halyard was configured consistent with where we want
# these BOMs to go.
self.__hal_runner.check_property('spinnaker.config.input.bucket',
options.halyard_bom_bucket)
def _do_repository(self, repository):
"""Implements RepositoryCommandProcessor interface."""
self.source_code_manager.ensure_local_repository(repository)
self.__collect_halconfig_files(repository)
def _do_postprocess(self, _):
"""Implements RepositoryCommandProcessor interface."""
options = self.options
bom_path = _determine_bom_path(self)
self.__hal_runner.publish_bom_path(bom_path)
self.__publish_configs(bom_path)
if options.bom_alias:
alias = options.bom_alias
logging.info('Publishing bom alias %s = %s', alias,
os.path.basename(bom_path))
with open(bom_path, 'r') as stream:
bom = yaml.load(stream)
alias_path = os.path.join(os.path.dirname(bom_path),
alias + '.yml')
with open(alias_path, 'w') as stream:
bom['version'] = options.bom_alias
yaml.dump(bom, stream, default_flow_style=False)
self.__hal_runner.publish_bom_path(alias_path)
def __publish_configs(self, bom_path):
"""Publish each of the halconfigs for the bom at the given path."""
def publish_repo_config(repository):
"""Helper function to publish individual repository."""
name = self.scm.repository_name_to_service_name(repository.name)
config_dir = os.path.join(self.get_output_dir(), 'halconfig', name)
if not os.path.exists(config_dir):
logging.warning('No profiles for %s', name)
return
logging.debug('Publishing profiles for %s', name)
for profile in os.listdir(config_dir):
profile_path = os.path.join(config_dir, profile)
self.__hal_runner.publish_profile(name, profile_path, bom_path)
logging.info('Publishing halyard configs...')
self.source_code_manager.foreach_source_repository(
self.source_repositories, publish_repo_config)
def __collect_halconfig_files(self, repository):
"""Gets the component config files and writes them into the output_dir."""
name = repository.name
if (name not in SPINNAKER_BOM_REPOSITORY_NAMES
or name in ['spinnaker', 'spin']):
logging.debug('%s does not use config files -- skipping', name)
return
if name == 'spinnaker-monitoring':
config_root = os.path.join(repository.git_dir,
'spinnaker-monitoring-daemon')
else:
config_root = repository.git_dir
service_name = self.scm.repository_name_to_service_name(
repository.name)
target_dir = os.path.join(self.get_output_dir(), 'halconfig',
service_name)
ensure_dir_exists(target_dir)
config_path = os.path.join(config_root, 'halconfig')
logging.info('Copying configs from %s...', config_path)
for profile in os.listdir(config_path):
profile_path = os.path.join(config_path, profile)
if os.path.isfile(profile_path):
shutil.copyfile(profile_path,
os.path.join(target_dir, profile))
logging.debug('Copied profile to %s', profile_path)
elif not os.path.isdir(profile_path):
logging.warning('%s is neither file nor directory -- ignoring',
profile_path)
continue
else:
tar_path = os.path.join(
target_dir, '{profile}.tar.gz'.format(profile=profile))
file_list = ' '.join(os.listdir(profile_path))
# NOTE: For historic reasons this is not actually compressed
# even though the tar_path says ".tar.gz"
check_subprocess(
'tar cf {path} -C {profile} {file_list}'.format(
path=tar_path,
profile=profile_path,
file_list=file_list))
logging.debug('Copied profile to %s', tar_path)
class PublishBomCommand(RepositoryCommandProcessor):
"""Implements publish_bom"""
def __init__(self, factory, options, **kwargs):
options.github_disable_upstream_push = True
super(PublishBomCommand, self).__init__(factory, options, **kwargs)
self.__hal_runner = HalRunner(options)
logging.debug('Verifying halyard server is consistent')
# Halyard is configured with fixed endpoints, however when we
# pubish we want to be explicit about where we are publishing to.
# There isnt a way to control this in halyard on a per-request basis
# so make sure halyard was configured consistent with where we want
# these BOMs to go.
self.__hal_runner.check_property(
'spinnaker.config.input.bucket', options.halyard_bom_bucket)
def _do_repository(self, repository):
"""Implements RepositoryCommandProcessor interface."""
self.source_code_manager.ensure_local_repository(repository)
self.__collect_halconfig_files(repository)
def _do_postprocess(self, _):
"""Implements RepositoryCommandProcessor interface."""
options = self.options
bom_path = _determine_bom_path(self)
self.__hal_runner.publish_bom_path(bom_path)
self.__publish_configs(bom_path)
if options.bom_alias:
alias = options.bom_alias
logging.info('Publishing bom alias %s = %s',
alias, os.path.basename(bom_path))
with open(bom_path, 'r') as stream:
bom = yaml.safe_load(stream)
alias_path = os.path.join(os.path.dirname(bom_path), alias + '.yml')
with open(alias_path, 'w') as stream:
bom['version'] = options.bom_alias
yaml.safe_dump(bom, stream, default_flow_style=False)
self.__hal_runner.publish_bom_path(alias_path)
def __publish_configs(self, bom_path):
"""Publish each of the halconfigs for the bom at the given path."""
def publish_repo_config(repository):
"""Helper function to publish individual repository."""
name = self.scm.repository_name_to_service_name(repository.name)
config_dir = os.path.join(self.get_output_dir(), 'halconfig', name)
if not os.path.exists(config_dir):
logging.warning('No profiles for %s', name)
return
logging.debug('Publishing profiles for %s', name)
for profile in os.listdir(config_dir):
profile_path = os.path.join(config_dir, profile)
self.__hal_runner.publish_profile(name, profile_path, bom_path)
logging.info('Publishing halyard configs...')
self.source_code_manager.foreach_source_repository(
self.source_repositories, publish_repo_config)
def __collect_halconfig_files(self, repository):
"""Gets the component config files and writes them into the output_dir."""
name = repository.name
if (name not in SPINNAKER_BOM_REPOSITORY_NAMES
or name in ['spin']):
logging.debug('%s does not use config files -- skipping', name)
return
if name == 'spinnaker-monitoring':
config_root = os.path.join(
repository.git_dir, 'spinnaker-monitoring-daemon')
else:
config_root = repository.git_dir
service_name = self.scm.repository_name_to_service_name(repository.name)
target_dir = os.path.join(self.get_output_dir(), 'halconfig', service_name)
ensure_dir_exists(target_dir)
config_path = os.path.join(config_root, 'halconfig')
logging.info('Copying configs from %s...', config_path)
for profile in os.listdir(config_path):
profile_path = os.path.join(config_path, profile)
if os.path.isfile(profile_path):
shutil.copyfile(profile_path, os.path.join(target_dir, profile))
logging.debug('Copied profile to %s', profile_path)
elif not os.path.isdir(profile_path):
logging.warning('%s is neither file nor directory -- ignoring',
profile_path)
continue
else:
tar_path = os.path.join(
target_dir, '{profile}.tar.gz'.format(profile=profile))
file_list = ' '.join(os.listdir(profile_path))
# NOTE: For historic reasons this is not actually compressed
# even though the tar_path says ".tar.gz"
check_subprocess(
'tar cf {path} -C {profile} {file_list}'.format(
path=tar_path, profile=profile_path, file_list=file_list))
logging.debug('Copied profile to %s', tar_path)
class PublishSpinnakerCommand(CommandProcessor):
""""Implements the publish_spinnaker command."""
# pylint: disable=too-few-public-methods
def __init__(self, factory, options, **kwargs):
super(PublishSpinnakerCommand, self).__init__(factory, options,
**kwargs)
check_options_set(options, [
'spinnaker_version', 'spinnaker_release_alias', 'bom_version',
'changelog_gist_url', 'github_owner', 'min_halyard_version'
])
major, minor, _ = self.options.spinnaker_version.split('.')
self.__branch = 'release-{major}.{minor}.x'.format(major=major,
minor=minor)
options_copy = copy.copy(options)
self.__bom_scm = BomSourceCodeManager(options_copy,
self.get_input_dir())
self.__hal = HalRunner(options)
self.__git = GitRunner(options)
self.__hal.check_property('spinnaker.config.input.bucket',
options.halyard_bom_bucket)
if options.only_repositories:
self.__only_repositories = options.only_repositories.split(',')
else:
self.__only_repositories = []
options_copy.git_branch = self.__branch
self.__branch_scm = BranchSourceCodeManager(options_copy,
self.get_input_dir())
def push_branches_and_tags(self, bom):
"""Update the release branches and tags in each of the BOM repositires."""
logging.info('Tagging each of the BOM service repos')
bom_scm = self.__bom_scm
branch_scm = self.__branch_scm
# Run in two passes so we dont push anything if we hit a problem
# in the tagging pass. Since we are spread against multiple repositiories,
# we cannot do this atomically. The two passes gives us more protection
# from a partial push due to errors in a repo.
names_to_push = set([])
for which in ['tag', 'push']:
for name, spec in bom['services'].items():
if name in ['monitoring-third-party', 'defaultArtifact']:
# Ignore this, it is redundant to monitoring-daemon
continue
if name == 'monitoring-daemon':
name = 'spinnaker-monitoring'
if self.__only_repositories and name not in self.__only_repositories:
logging.debug('Skipping %s because of --only_repositories',
name)
continue
if spec is None:
logging.warning('HAVE bom.services.%s = None', name)
continue
repository = bom_scm.make_repository_spec(name)
bom_scm.ensure_local_repository(repository)
version = bom_scm.determine_repository_version(repository)
if which == 'tag':
added = self.__branch_and_tag_repository(
repository, self.__branch, version)
if added:
names_to_push.add(name)
else:
self.__push_branch_and_maybe_tag_repository(
repository, self.__branch, version, name
in names_to_push)
additional_repositories = list(SPINNAKER_PROCESS_REPOSITORY_NAMES)
for name in additional_repositories:
if self.__only_repositories and name not in self.__only_repositories:
logging.debug('Skipping %s because of --only_repositories',
name)
continue
repository = branch_scm.make_repository_spec(name)
branch_scm.ensure_local_repository(repository)
git_summary = self.__git.collect_repository_summary(
repository.git_dir)
version = git_summary.version
if self.__branch_and_tag_repository(repository, self.__branch,
version):
self.__push_branch_and_maybe_tag_repository(
repository, self.__branch, version, True)
def __already_have_tag(self, repository, tag):
"""Determine if we already have the tag in the repository."""
git_dir = repository.git_dir
existing_commit = self.__git.query_commit_at_tag(git_dir, tag)
if not existing_commit:
return False
want_commit = self.__git.query_local_repository_commit_id(git_dir)
if want_commit == existing_commit:
logging.debug('Already have "%s" at %s', tag, want_commit)
return True
raise_and_log_error(
ConfigError(
'"{tag}" already exists in "{repo}" at commit {have}, not {want}'
.format(tag=tag,
repo=git_dir,
have=existing_commit,
want=want_commit)))
def __branch_and_tag_repository(self, repository, branch, version):
"""Create a branch and/or verison tag in the repository, if needed."""
tag = 'version-' + version
if self.__already_have_tag(repository, tag):
return False
self.__git.check_run(repository.git_dir, 'tag ' + tag)
return True
def __push_branch_and_maybe_tag_repository(self, repository, branch,
version, also_tag):
"""Push the branch and verison tag to the origin."""
tag = 'version-' + version
self.__git.push_branch_to_origin(repository.git_dir, branch)
if also_tag:
self.__git.push_tag_to_origin(repository.git_dir, tag)
else:
logging.info('%s was already tagged with "%s" -- skip',
repository.git_dir, tag)
def _do_command(self):
"""Implements CommandProcessor interface."""
options = self.options
spinnaker_version = options.spinnaker_version
options_copy = copy.copy(options)
options_copy.git_branch = 'master' # push to master in spinnaker.github.io
publish_changelog_command = PublishChangelogFactory().make_command(
options_copy)
changelog_gist_url = options.changelog_gist_url
# Make sure changelog exists already.
# If it does not then fail.
try:
logging.debug('Verifying changelog ready at %s',
changelog_gist_url)
urlopen(changelog_gist_url)
except HTTPError:
logging.error(exception_to_message)
raise_and_log_error(
ConfigError(
'Changelog gist "{url}" must exist before publising a release.'
.format(url=changelog_gist_url),
cause='ChangelogMissing'))
bom = self.__hal.retrieve_bom_version(self.options.bom_version)
bom['version'] = spinnaker_version
bom_path = os.path.join(self.get_output_dir(),
spinnaker_version + '.yml')
write_to_path(yaml.safe_dump(bom, default_flow_style=False), bom_path)
self.__hal.publish_bom_path(bom_path)
self.push_branches_and_tags(bom)
self.__hal.publish_spinnaker_release(spinnaker_version,
options.spinnaker_release_alias,
changelog_gist_url,
options.min_halyard_version)
logging.info('Publishing changelog')
publish_changelog_command()
class PublishSpinnakerCommand(CommandProcessor):
""""Implements the publish_spinnaker command."""
# pylint: disable=too-few-public-methods
def __init__(self, factory, options, **kwargs):
super(PublishSpinnakerCommand, self).__init__(factory, options, **kwargs)
check_options_set(options, [
'spinnaker_version',
'spinnaker_release_alias',
'bom_version',
'changelog_gist_url',
'github_owner',
'min_halyard_version'
])
major, minor, _ = self.options.spinnaker_version.split('.')
self.__branch = 'release-{major}.{minor}.x'.format(
major=major, minor=minor)
options_copy = copy.copy(options)
self.__bom_scm = BomSourceCodeManager(options_copy, self.get_input_dir())
self.__hal = HalRunner(options)
self.__git = GitRunner(options)
self.__hal.check_property(
'spinnaker.config.input.bucket', options.halyard_bom_bucket)
if options.only_repositories:
self.__only_repositories = options.only_repositories.split(',')
else:
self.__only_repositories = []
options_copy.git_branch = self.__branch
self.__branch_scm = BranchSourceCodeManager(
options_copy, self.get_input_dir())
def push_branches_and_tags(self, bom):
"""Update the release branches and tags in each of the BOM repositires."""
logging.info('Tagging each of the BOM service repos')
bom_scm = self.__bom_scm
branch_scm = self.__branch_scm
# Run in two passes so we dont push anything if we hit a problem
# in the tagging pass. Since we are spread against multiple repositiories,
# we cannot do this atomically. The two passes gives us more protection
# from a partial push due to errors in a repo.
names_to_push = set([])
for which in ['tag', 'push']:
for name, spec in bom['services'].items():
if name in ['monitoring-third-party', 'defaultArtifact']:
# Ignore this, it is redundant to monitoring-daemon
continue
if name == 'monitoring-daemon':
name = 'spinnaker-monitoring'
if self.__only_repositories and name not in self.__only_repositories:
logging.debug('Skipping %s because of --only_repositories', name)
continue
if spec is None:
logging.warning('HAVE bom.services.%s = None', name)
continue
repository = bom_scm.make_repository_spec(name)
bom_scm.ensure_local_repository(repository)
version = bom_scm.determine_repository_version(repository)
if which == 'tag':
added = self.__branch_and_tag_repository(
repository, self.__branch, version)
if added:
names_to_push.add(name)
else:
self.__push_branch_and_maybe_tag_repository(
repository, self.__branch, version, name in names_to_push)
additional_repositories = list(SPINNAKER_PROCESS_REPOSITORY_NAMES)
for name in additional_repositories:
if self.__only_repositories and name not in self.__only_repositories:
logging.debug('Skipping %s because of --only_repositories', name)
continue
repository = branch_scm.make_repository_spec(name)
branch_scm.ensure_local_repository(repository)
git_summary = self.__git.collect_repository_summary(repository.git_dir)
version = git_summary.version
if self.__branch_and_tag_repository(
repository, self.__branch, version):
self.__push_branch_and_maybe_tag_repository(
repository, self.__branch, version, True)
def __already_have_tag(self, repository, tag):
"""Determine if we already have the tag in the repository."""
git_dir = repository.git_dir
existing_commit = self.__git.query_commit_at_tag(git_dir, tag)
if not existing_commit:
return False
want_commit = self.__git.query_local_repository_commit_id(git_dir)
if want_commit == existing_commit:
logging.debug('Already have "%s" at %s', tag, want_commit)
return True
raise_and_log_error(
ConfigError(
'"{tag}" already exists in "{repo}" at commit {have}, not {want}'
.format(tag=tag, repo=git_dir,
have=existing_commit, want=want_commit)))
return False # not reached
def __branch_and_tag_repository(self, repository, branch, version):
"""Create a branch and/or version tag in the repository, if needed."""
tag = 'version-' + version
if self.__already_have_tag(repository, tag):
return False
self.__git.check_run(repository.git_dir, 'tag ' + tag)
return True
def __push_branch_and_maybe_tag_repository(self, repository, branch, version,
also_tag):
"""Push the branch and version tag to the origin."""
tag = 'version-' + version
self.__git.push_branch_to_origin(repository.git_dir, branch)
if also_tag:
self.__git.push_tag_to_origin(repository.git_dir, tag)
else:
logging.info('%s was already tagged with "%s" -- skip',
repository.git_dir, tag)
def _do_command(self):
"""Implements CommandProcessor interface."""
options = self.options
spinnaker_version = options.spinnaker_version
options_copy = copy.copy(options)
options_copy.git_branch = 'master' # push to master in spinnaker.github.io
publish_changelog_command = PublishChangelogFactory().make_command(
options_copy)
changelog_gist_url = options.changelog_gist_url
# Make sure changelog exists already.
# If it does not then fail.
try:
logging.debug('Verifying changelog ready at %s', changelog_gist_url)
urlopen(changelog_gist_url)
except HTTPError:
logging.error(exception_to_message)
raise_and_log_error(
ConfigError(
'Changelog gist "{url}" must exist before publising a release.'
.format(url=changelog_gist_url),
cause='ChangelogMissing'))
bom = self.__hal.retrieve_bom_version(self.options.bom_version)
bom['version'] = spinnaker_version
bom_path = os.path.join(self.get_output_dir(), spinnaker_version + '.yml')
write_to_path(yaml.safe_dump(bom, default_flow_style=False), bom_path)
self.__hal.publish_bom_path(bom_path)
self.push_branches_and_tags(bom)
self.__hal.publish_spinnaker_release(
spinnaker_version, options.spinnaker_release_alias, changelog_gist_url,
options.min_halyard_version)
logging.info('Publishing changelog')
publish_changelog_command()
