class PublishBomCommand(RepositoryCommandProcessor): """Implements publish_bom""" def __init__(self, factory, options, **kwargs): options.github_disable_upstream_push = True super(PublishBomCommand, self).__init__(factory, options, **kwargs) self.__hal_runner = HalRunner(options) logging.debug('Verifying halyard server is consistent') # Halyard is configured with fixed endpoints, however when we # pubish we want to be explicit about where we are publishing to. # There isnt a way to control this in halyard on a per-request basis # so make sure halyard was configured consistent with where we want # these BOMs to go. self.__hal_runner.check_property('spinnaker.config.input.bucket', options.halyard_bom_bucket) def _do_repository(self, repository): """Implements RepositoryCommandProcessor interface.""" self.source_code_manager.ensure_local_repository(repository) self.__collect_halconfig_files(repository) def _do_postprocess(self, _): """Implements RepositoryCommandProcessor interface.""" options = self.options bom_path = _determine_bom_path(self) self.__hal_runner.publish_bom_path(bom_path) self.__publish_configs(bom_path) if options.bom_alias: alias = options.bom_alias logging.info('Publishing bom alias %s = %s', alias, os.path.basename(bom_path)) with open(bom_path, 'r') as stream: bom = yaml.load(stream) alias_path = os.path.join(os.path.dirname(bom_path), alias + '.yml') with open(alias_path, 'w') as stream: bom['version'] = options.bom_alias yaml.dump(bom, stream, default_flow_style=False) self.__hal_runner.publish_bom_path(alias_path) def __publish_configs(self, bom_path): """Publish each of the halconfigs for the bom at the given path.""" def publish_repo_config(repository): """Helper function to publish individual repository.""" name = self.scm.repository_name_to_service_name(repository.name) config_dir = os.path.join(self.get_output_dir(), 'halconfig', name) if not os.path.exists(config_dir): logging.warning('No profiles for %s', name) return logging.debug('Publishing profiles for %s', name) for profile in os.listdir(config_dir): profile_path = os.path.join(config_dir, profile) self.__hal_runner.publish_profile(name, profile_path, bom_path) logging.info('Publishing halyard configs...') self.source_code_manager.foreach_source_repository( self.source_repositories, publish_repo_config) def __collect_halconfig_files(self, repository): """Gets the component config files and writes them into the output_dir.""" name = repository.name if (name not in SPINNAKER_BOM_REPOSITORY_NAMES or name in ['spinnaker', 'spin']): logging.debug('%s does not use config files -- skipping', name) return if name == 'spinnaker-monitoring': config_root = os.path.join(repository.git_dir, 'spinnaker-monitoring-daemon') else: config_root = repository.git_dir service_name = self.scm.repository_name_to_service_name( repository.name) target_dir = os.path.join(self.get_output_dir(), 'halconfig', service_name) ensure_dir_exists(target_dir) config_path = os.path.join(config_root, 'halconfig') logging.info('Copying configs from %s...', config_path) for profile in os.listdir(config_path): profile_path = os.path.join(config_path, profile) if os.path.isfile(profile_path): shutil.copyfile(profile_path, os.path.join(target_dir, profile)) logging.debug('Copied profile to %s', profile_path) elif not os.path.isdir(profile_path): logging.warning('%s is neither file nor directory -- ignoring', profile_path) continue else: tar_path = os.path.join( target_dir, '{profile}.tar.gz'.format(profile=profile)) file_list = ' '.join(os.listdir(profile_path)) # NOTE: For historic reasons this is not actually compressed # even though the tar_path says ".tar.gz" check_subprocess( 'tar cf {path} -C {profile} {file_list}'.format( path=tar_path, profile=profile_path, file_list=file_list)) logging.debug('Copied profile to %s', tar_path)
class PublishBomCommand(RepositoryCommandProcessor): """Implements publish_bom""" def __init__(self, factory, options, **kwargs): options.github_disable_upstream_push = True super(PublishBomCommand, self).__init__(factory, options, **kwargs) self.__hal_runner = HalRunner(options) logging.debug('Verifying halyard server is consistent') # Halyard is configured with fixed endpoints, however when we # pubish we want to be explicit about where we are publishing to. # There isnt a way to control this in halyard on a per-request basis # so make sure halyard was configured consistent with where we want # these BOMs to go. self.__hal_runner.check_property( 'spinnaker.config.input.bucket', options.halyard_bom_bucket) def _do_repository(self, repository): """Implements RepositoryCommandProcessor interface.""" self.source_code_manager.ensure_local_repository(repository) self.__collect_halconfig_files(repository) def _do_postprocess(self, _): """Implements RepositoryCommandProcessor interface.""" options = self.options bom_path = _determine_bom_path(self) self.__hal_runner.publish_bom_path(bom_path) self.__publish_configs(bom_path) if options.bom_alias: alias = options.bom_alias logging.info('Publishing bom alias %s = %s', alias, os.path.basename(bom_path)) with open(bom_path, 'r') as stream: bom = yaml.safe_load(stream) alias_path = os.path.join(os.path.dirname(bom_path), alias + '.yml') with open(alias_path, 'w') as stream: bom['version'] = options.bom_alias yaml.safe_dump(bom, stream, default_flow_style=False) self.__hal_runner.publish_bom_path(alias_path) def __publish_configs(self, bom_path): """Publish each of the halconfigs for the bom at the given path.""" def publish_repo_config(repository): """Helper function to publish individual repository.""" name = self.scm.repository_name_to_service_name(repository.name) config_dir = os.path.join(self.get_output_dir(), 'halconfig', name) if not os.path.exists(config_dir): logging.warning('No profiles for %s', name) return logging.debug('Publishing profiles for %s', name) for profile in os.listdir(config_dir): profile_path = os.path.join(config_dir, profile) self.__hal_runner.publish_profile(name, profile_path, bom_path) logging.info('Publishing halyard configs...') self.source_code_manager.foreach_source_repository( self.source_repositories, publish_repo_config) def __collect_halconfig_files(self, repository): """Gets the component config files and writes them into the output_dir.""" name = repository.name if (name not in SPINNAKER_BOM_REPOSITORY_NAMES or name in ['spin']): logging.debug('%s does not use config files -- skipping', name) return if name == 'spinnaker-monitoring': config_root = os.path.join( repository.git_dir, 'spinnaker-monitoring-daemon') else: config_root = repository.git_dir service_name = self.scm.repository_name_to_service_name(repository.name) target_dir = os.path.join(self.get_output_dir(), 'halconfig', service_name) ensure_dir_exists(target_dir) config_path = os.path.join(config_root, 'halconfig') logging.info('Copying configs from %s...', config_path) for profile in os.listdir(config_path): profile_path = os.path.join(config_path, profile) if os.path.isfile(profile_path): shutil.copyfile(profile_path, os.path.join(target_dir, profile)) logging.debug('Copied profile to %s', profile_path) elif not os.path.isdir(profile_path): logging.warning('%s is neither file nor directory -- ignoring', profile_path) continue else: tar_path = os.path.join( target_dir, '{profile}.tar.gz'.format(profile=profile)) file_list = ' '.join(os.listdir(profile_path)) # NOTE: For historic reasons this is not actually compressed # even though the tar_path says ".tar.gz" check_subprocess( 'tar cf {path} -C {profile} {file_list}'.format( path=tar_path, profile=profile_path, file_list=file_list)) logging.debug('Copied profile to %s', tar_path)
class PublishSpinnakerCommand(CommandProcessor): """"Implements the publish_spinnaker command.""" # pylint: disable=too-few-public-methods def __init__(self, factory, options, **kwargs): super(PublishSpinnakerCommand, self).__init__(factory, options, **kwargs) check_options_set(options, [ 'spinnaker_version', 'spinnaker_release_alias', 'bom_version', 'changelog_gist_url', 'github_owner', 'min_halyard_version' ]) major, minor, _ = self.options.spinnaker_version.split('.') self.__branch = 'release-{major}.{minor}.x'.format(major=major, minor=minor) options_copy = copy.copy(options) self.__bom_scm = BomSourceCodeManager(options_copy, self.get_input_dir()) self.__hal = HalRunner(options) self.__git = GitRunner(options) self.__hal.check_property('spinnaker.config.input.bucket', options.halyard_bom_bucket) if options.only_repositories: self.__only_repositories = options.only_repositories.split(',') else: self.__only_repositories = [] options_copy.git_branch = self.__branch self.__branch_scm = BranchSourceCodeManager(options_copy, self.get_input_dir()) def push_branches_and_tags(self, bom): """Update the release branches and tags in each of the BOM repositires.""" logging.info('Tagging each of the BOM service repos') bom_scm = self.__bom_scm branch_scm = self.__branch_scm # Run in two passes so we dont push anything if we hit a problem # in the tagging pass. Since we are spread against multiple repositiories, # we cannot do this atomically. The two passes gives us more protection # from a partial push due to errors in a repo. names_to_push = set([]) for which in ['tag', 'push']: for name, spec in bom['services'].items(): if name in ['monitoring-third-party', 'defaultArtifact']: # Ignore this, it is redundant to monitoring-daemon continue if name == 'monitoring-daemon': name = 'spinnaker-monitoring' if self.__only_repositories and name not in self.__only_repositories: logging.debug('Skipping %s because of --only_repositories', name) continue if spec is None: logging.warning('HAVE bom.services.%s = None', name) continue repository = bom_scm.make_repository_spec(name) bom_scm.ensure_local_repository(repository) version = bom_scm.determine_repository_version(repository) if which == 'tag': added = self.__branch_and_tag_repository( repository, self.__branch, version) if added: names_to_push.add(name) else: self.__push_branch_and_maybe_tag_repository( repository, self.__branch, version, name in names_to_push) additional_repositories = list(SPINNAKER_PROCESS_REPOSITORY_NAMES) for name in additional_repositories: if self.__only_repositories and name not in self.__only_repositories: logging.debug('Skipping %s because of --only_repositories', name) continue repository = branch_scm.make_repository_spec(name) branch_scm.ensure_local_repository(repository) git_summary = self.__git.collect_repository_summary( repository.git_dir) version = git_summary.version if self.__branch_and_tag_repository(repository, self.__branch, version): self.__push_branch_and_maybe_tag_repository( repository, self.__branch, version, True) def __already_have_tag(self, repository, tag): """Determine if we already have the tag in the repository.""" git_dir = repository.git_dir existing_commit = self.__git.query_commit_at_tag(git_dir, tag) if not existing_commit: return False want_commit = self.__git.query_local_repository_commit_id(git_dir) if want_commit == existing_commit: logging.debug('Already have "%s" at %s', tag, want_commit) return True raise_and_log_error( ConfigError( '"{tag}" already exists in "{repo}" at commit {have}, not {want}' .format(tag=tag, repo=git_dir, have=existing_commit, want=want_commit))) def __branch_and_tag_repository(self, repository, branch, version): """Create a branch and/or verison tag in the repository, if needed.""" tag = 'version-' + version if self.__already_have_tag(repository, tag): return False self.__git.check_run(repository.git_dir, 'tag ' + tag) return True def __push_branch_and_maybe_tag_repository(self, repository, branch, version, also_tag): """Push the branch and verison tag to the origin.""" tag = 'version-' + version self.__git.push_branch_to_origin(repository.git_dir, branch) if also_tag: self.__git.push_tag_to_origin(repository.git_dir, tag) else: logging.info('%s was already tagged with "%s" -- skip', repository.git_dir, tag) def _do_command(self): """Implements CommandProcessor interface.""" options = self.options spinnaker_version = options.spinnaker_version options_copy = copy.copy(options) options_copy.git_branch = 'master' # push to master in spinnaker.github.io publish_changelog_command = PublishChangelogFactory().make_command( options_copy) changelog_gist_url = options.changelog_gist_url # Make sure changelog exists already. # If it does not then fail. try: logging.debug('Verifying changelog ready at %s', changelog_gist_url) urlopen(changelog_gist_url) except HTTPError: logging.error(exception_to_message) raise_and_log_error( ConfigError( 'Changelog gist "{url}" must exist before publising a release.' .format(url=changelog_gist_url), cause='ChangelogMissing')) bom = self.__hal.retrieve_bom_version(self.options.bom_version) bom['version'] = spinnaker_version bom_path = os.path.join(self.get_output_dir(), spinnaker_version + '.yml') write_to_path(yaml.safe_dump(bom, default_flow_style=False), bom_path) self.__hal.publish_bom_path(bom_path) self.push_branches_and_tags(bom) self.__hal.publish_spinnaker_release(spinnaker_version, options.spinnaker_release_alias, changelog_gist_url, options.min_halyard_version) logging.info('Publishing changelog') publish_changelog_command()
class PublishSpinnakerCommand(CommandProcessor): """"Implements the publish_spinnaker command.""" # pylint: disable=too-few-public-methods def __init__(self, factory, options, **kwargs): super(PublishSpinnakerCommand, self).__init__(factory, options, **kwargs) check_options_set(options, [ 'spinnaker_version', 'spinnaker_release_alias', 'bom_version', 'changelog_gist_url', 'github_owner', 'min_halyard_version' ]) major, minor, _ = self.options.spinnaker_version.split('.') self.__branch = 'release-{major}.{minor}.x'.format( major=major, minor=minor) options_copy = copy.copy(options) self.__bom_scm = BomSourceCodeManager(options_copy, self.get_input_dir()) self.__hal = HalRunner(options) self.__git = GitRunner(options) self.__hal.check_property( 'spinnaker.config.input.bucket', options.halyard_bom_bucket) if options.only_repositories: self.__only_repositories = options.only_repositories.split(',') else: self.__only_repositories = [] options_copy.git_branch = self.__branch self.__branch_scm = BranchSourceCodeManager( options_copy, self.get_input_dir()) def push_branches_and_tags(self, bom): """Update the release branches and tags in each of the BOM repositires.""" logging.info('Tagging each of the BOM service repos') bom_scm = self.__bom_scm branch_scm = self.__branch_scm # Run in two passes so we dont push anything if we hit a problem # in the tagging pass. Since we are spread against multiple repositiories, # we cannot do this atomically. The two passes gives us more protection # from a partial push due to errors in a repo. names_to_push = set([]) for which in ['tag', 'push']: for name, spec in bom['services'].items(): if name in ['monitoring-third-party', 'defaultArtifact']: # Ignore this, it is redundant to monitoring-daemon continue if name == 'monitoring-daemon': name = 'spinnaker-monitoring' if self.__only_repositories and name not in self.__only_repositories: logging.debug('Skipping %s because of --only_repositories', name) continue if spec is None: logging.warning('HAVE bom.services.%s = None', name) continue repository = bom_scm.make_repository_spec(name) bom_scm.ensure_local_repository(repository) version = bom_scm.determine_repository_version(repository) if which == 'tag': added = self.__branch_and_tag_repository( repository, self.__branch, version) if added: names_to_push.add(name) else: self.__push_branch_and_maybe_tag_repository( repository, self.__branch, version, name in names_to_push) additional_repositories = list(SPINNAKER_PROCESS_REPOSITORY_NAMES) for name in additional_repositories: if self.__only_repositories and name not in self.__only_repositories: logging.debug('Skipping %s because of --only_repositories', name) continue repository = branch_scm.make_repository_spec(name) branch_scm.ensure_local_repository(repository) git_summary = self.__git.collect_repository_summary(repository.git_dir) version = git_summary.version if self.__branch_and_tag_repository( repository, self.__branch, version): self.__push_branch_and_maybe_tag_repository( repository, self.__branch, version, True) def __already_have_tag(self, repository, tag): """Determine if we already have the tag in the repository.""" git_dir = repository.git_dir existing_commit = self.__git.query_commit_at_tag(git_dir, tag) if not existing_commit: return False want_commit = self.__git.query_local_repository_commit_id(git_dir) if want_commit == existing_commit: logging.debug('Already have "%s" at %s', tag, want_commit) return True raise_and_log_error( ConfigError( '"{tag}" already exists in "{repo}" at commit {have}, not {want}' .format(tag=tag, repo=git_dir, have=existing_commit, want=want_commit))) return False # not reached def __branch_and_tag_repository(self, repository, branch, version): """Create a branch and/or version tag in the repository, if needed.""" tag = 'version-' + version if self.__already_have_tag(repository, tag): return False self.__git.check_run(repository.git_dir, 'tag ' + tag) return True def __push_branch_and_maybe_tag_repository(self, repository, branch, version, also_tag): """Push the branch and version tag to the origin.""" tag = 'version-' + version self.__git.push_branch_to_origin(repository.git_dir, branch) if also_tag: self.__git.push_tag_to_origin(repository.git_dir, tag) else: logging.info('%s was already tagged with "%s" -- skip', repository.git_dir, tag) def _do_command(self): """Implements CommandProcessor interface.""" options = self.options spinnaker_version = options.spinnaker_version options_copy = copy.copy(options) options_copy.git_branch = 'master' # push to master in spinnaker.github.io publish_changelog_command = PublishChangelogFactory().make_command( options_copy) changelog_gist_url = options.changelog_gist_url # Make sure changelog exists already. # If it does not then fail. try: logging.debug('Verifying changelog ready at %s', changelog_gist_url) urlopen(changelog_gist_url) except HTTPError: logging.error(exception_to_message) raise_and_log_error( ConfigError( 'Changelog gist "{url}" must exist before publising a release.' .format(url=changelog_gist_url), cause='ChangelogMissing')) bom = self.__hal.retrieve_bom_version(self.options.bom_version) bom['version'] = spinnaker_version bom_path = os.path.join(self.get_output_dir(), spinnaker_version + '.yml') write_to_path(yaml.safe_dump(bom, default_flow_style=False), bom_path) self.__hal.publish_bom_path(bom_path) self.push_branches_and_tags(bom) self.__hal.publish_spinnaker_release( spinnaker_version, options.spinnaker_release_alias, changelog_gist_url, options.min_halyard_version) logging.info('Publishing changelog') publish_changelog_command()