def make_default_options(): """Helper function for creating default options for runner.""" parser = argparse.ArgumentParser() GitRunner.add_parser_args(parser, {'github_disable_upstream_push': True}) parser.add_argument('--output_dir', default=os.path.join('/tmp', 'gittest.%d' % os.getpid())) return parser.parse_args([])
def to_git_url_prefix(self, url): """Determine url up to the terminal path component.""" if url.startswith('git@'): parts = GitRunner.normalize_repo_url(url) url = GitRunner.make_https_url(*parts) # We're assuming no query parameter/fragment since these are git URLs. # otherwise we need to parse the url and extract the path return url[:url.rfind('/')]
def add_parser_args(parser, defaults): """Add standard parser arguments used by SourceCodeManager.""" if hasattr(parser, 'added_scm'): return parser.added_scm = True GitRunner.add_parser_args(parser, defaults) add_parser_argument(parser, 'github_upstream_owner', defaults, 'spinnaker', help='The standard upstream repository owner.')
def main(): """The main command dispatcher.""" start_time = time.time() from importlib import import_module command_modules = [ import_module(name + '_commands') for name in [ 'apidocs', 'bom', 'changelog', 'container', 'debian', 'halyard', 'image', 'rpm', 'source', 'spinnaker', 'inspection', 'spin', ]] GitRunner.stash_and_clear_auth_env_vars() options, command_registry = init_options_and_registry( sys.argv[1:], command_modules) logging.basicConfig( format='%(levelname).1s %(asctime)s.%(msecs)03d' ' [%(threadName)s.%(process)d] %(message)s', datefmt='%H:%M:%S', level=STANDARD_LOG_LEVELS[options.log_level]) logging.debug( 'Running with options:\n %s', '\n '.join(yaml.safe_dump(vars(options), default_flow_style=False) .split('\n'))) factory = command_registry.get(options.command) if not factory: logging.error('Unknown command "%s"', options.command) return -1 MetricsManager.startup_metrics(options) labels = {'command': options.command} success = False try: command = factory.make_command(options) command() success = True finally: labels['success'] = success MetricsManager.singleton().observe_timer( 'BuildTool_Outcome', labels, time.time() - start_time) MetricsManager.shutdown_metrics() return 0
def add_parser_args(parser, defaults): """Add standard parser arguments used by SourceCodeManager.""" if hasattr(parser, 'added_branch_scm'): return parser.added_branch_scm = True SpinnakerSourceCodeManager.add_parser_args(parser, defaults) GitRunner.add_parser_args(parser, defaults) add_parser_argument(parser, 'git_branch', defaults, None, help='The git branch to operate on.') add_parser_argument(parser, 'github_hostname', defaults, 'github.com', help='The hostname of the git server.')
def init_argparser(self, parser, defaults): """Adds command-specific arguments.""" super(PublishApiDocsFactory, self).init_argparser( parser, defaults) GitRunner.add_parser_args(parser, defaults) GitRunner.add_publishing_parser_args(parser, defaults) self.add_argument( parser, 'git_branch', defaults, None, help='The branch to checkout in ' + SPINNAKER_GITHUB_IO_REPOSITORY_NAME) self.add_argument( parser, 'spinnaker_version', defaults, None, help='The version of spinnaker this documentation is for.')
def init_argparser(self, parser, defaults): super(PublishChangelogFactory, self).init_argparser( parser, defaults) GitRunner.add_parser_args(parser, defaults) GitRunner.add_publishing_parser_args(parser, defaults) self.add_argument( parser, 'spinnaker_version', defaults, None, help='The version of spinnaker this documentation is for.') self.add_argument( parser, 'changelog_gist_url', defaults, None, help='The gist to the existing changelog content being published.')
def init_argparser(self, parser, defaults): GitRunner.add_parser_args(parser, defaults) GitRunner.add_publishing_parser_args(parser, defaults) super(InitiateReleaseBranchFactory, self).init_argparser(parser, defaults) self.add_argument( parser, 'skip_existing', defaults, False, type=bool, help='Leave the existing tag if found in a repository.') self.add_argument( parser, 'delete_existing', defaults, False, type=bool, help='Delete the existing tag if found in a repository.') self.add_argument( parser, 'spinnaker_version', defaults, None, help='The version branch name should be "release-<num>.<num>.x"')
def __init__(self, factory, options, **kwargs): super(PublishSpinnakerCommand, self).__init__(factory, options, **kwargs) check_options_set(options, [ 'spinnaker_version', 'spinnaker_release_alias', 'bom_version', 'changelog_gist_url', 'github_owner', 'min_halyard_version' ]) major, minor, _ = self.options.spinnaker_version.split('.') self.__branch = 'release-{major}.{minor}.x'.format( major=major, minor=minor) options_copy = copy.copy(options) self.__bom_scm = BomSourceCodeManager(options_copy, self.get_input_dir()) self.__hal = HalRunner(options) self.__git = GitRunner(options) self.__hal.check_property( 'spinnaker.config.input.bucket', options.halyard_bom_bucket) if options.only_repositories: self.__only_repositories = options.only_repositories.split(',') else: self.__only_repositories = [] options_copy.git_branch = self.__branch self.__branch_scm = BranchSourceCodeManager( options_copy, self.get_input_dir())
def init_argparser(self, parser, defaults): GitRunner.add_parser_args(parser, defaults) GitRunner.add_publishing_parser_args(parser, defaults) super(PublishSpinCommandFactory, self).init_argparser( parser, defaults) self.add_argument( parser, 'spin_bucket', defaults, None, help='The bucket to publish spin binaries to.') self.add_argument( parser, 'spin_credentials_path', defaults, None, help='The credentials to use to authenticate with the bucket.') self.add_argument( parser, 'spin_version', defaults, None, help='The semantic version of the release to publish.') # BomSourceCodeManager adds bom_version and bom_path arguments to fetch BOMs. BomSourceCodeManager.add_parser_args(parser, defaults)
def init_argparser(self, parser, defaults): super(PushChangelogFactory, self).init_argparser( parser, defaults) GitRunner.add_parser_args(parser, defaults) self.add_argument( parser, 'changelog_path', defaults, None, help='The path to the changelog to push.') self.add_argument( parser, 'git_branch', defaults, None, help='The branch name that this changelog is for. Note that this does' ' not actually *use* any branches, rather the branch name is used' ' to decorates the changelog filename.') self.add_argument( parser, 'build_changelog_gist_url', defaults, None, help='The gist to push the changelog into.')
def __init__(self, options, root_source_dir, **kwargs): self.__max_threads = kwargs.pop('max_threads', 100) self.__add_upstream = kwargs.pop('attach_upstream', False) check_kwargs_empty(kwargs) self.__options = options self.__git = GitRunner(options) self.__root_source_dir = root_source_dir
def init_argparser(self, parser, defaults): super(PublishHalyardCommandFactory, self).init_argparser( parser, defaults) GradleCommandFactory.add_bom_parser_args(parser, defaults) SpinnakerSourceCodeManager.add_parser_args(parser, defaults) GradleRunner.add_parser_args(parser, defaults) GitRunner.add_publishing_parser_args(parser, defaults) HalRunner.add_parser_args(parser, defaults) self.add_argument( parser, 'build_number', defaults, DEFAULT_BUILD_NUMBER, help='Publishing halyard requires a rebuild. This is the build number' ' to use when rebuilding halyard.') self.add_argument( parser, 'halyard_version', defaults, None, help='The semantic version of the release to publish.') self.add_argument( parser, 'halyard_version_commits_url', defaults, None, help='URL to file containing version and git commit for successful' ' nightly builds. By default this will be' ' "{filename}" in the' ' --halyard_bucket_base_url.'.format( filename=BuildHalyardCommand.HALYARD_VERSIONS_BASENAME)) self.add_argument( parser, 'halyard_docker_image_base', defaults, None, help='Base Docker image name for writing halyard builds.') self.add_argument( parser, 'halyard_bucket_base_url', defaults, None, help='Base Google Cloud Storage URL for writing halyard builds.') self.add_argument(parser, 'docs_repo_owner', defaults, None, help='Owner of the docs repo if one was' ' specified. The default is --github_owner.') self.add_argument( parser, 'skip_existing', defaults, False, type=bool, help='Skip builds if the desired version already exists on bintray.') self.add_argument( parser, 'delete_existing', defaults, None, type=bool, help='Delete pre-existing desired versions if from bintray.')
def init_argparser(self, parser, defaults): super(PublishSpinnakerFactory, self).init_argparser(parser, defaults) HalRunner.add_parser_args(parser, defaults) GitRunner.add_parser_args(parser, defaults) GitRunner.add_publishing_parser_args(parser, defaults) PublishChangelogFactory().init_argparser(parser, defaults) self.add_argument( parser, 'spinnaker_release_alias', defaults, None, help='The spinnaker version alias to publish as.') self.add_argument( parser, 'halyard_bom_bucket', defaults, 'halconfig', help='The bucket manaing halyard BOMs and config profiles.') self.add_argument( parser, 'bom_version', defaults, None, help='The existing bom version usef for this release.') self.add_argument( parser, 'min_halyard_version', defaults, None, help='The minimum halyard version required.')
def __init__(self, factory, options, **kwargs): super(PushChangelogCommand, self).__init__(factory, options, **kwargs) check_options_set( options, ['build_changelog_gist_url', 'git_branch']) if not options.changelog_path: options.changelog_path = os.path.join( self.get_output_dir(command=BUILD_CHANGELOG_COMMAND), 'changelog.md') check_path_exists(options.changelog_path, why='changelog_path') self.__git = GitRunner(options)
def test_is_same_repo(self): variants = [ 'http://github.com/user/spinnaker', 'http://github.com/user/spinnaker.git', 'https://github.com/user/spinnaker', 'https://github.com/user/spinnaker.git', '[email protected]:user/spinnaker.git', '[email protected]:user/spinnaker.git' ] for url in variants: self.assertTrue(GitRunner.is_same_repo(variants[0], url))
def test_different_repo(self): variants = [ 'http://github.com/user/spinnaker', 'http://github.com/path/user/spinnaker', 'http://github.com/user/spinnaker/path', 'http://github.com/user/spinnaker.github', 'http://github/user/spinnaker', 'http://mydomain.com/user/spinnaker', 'path/user/spinnaker' ] for url in variants[1:]: self.assertFalse(GitRunner.is_same_repo(variants[0], url))
class InitiateReleaseBranchCommand(RepositoryCommandProcessor): def __init__(self, factory, options, **kwargs): super(InitiateReleaseBranchCommand, self).__init__( factory, options, **kwargs) check_options_set(options, ['spinnaker_version']) self.__git = GitRunner(options) def _do_repository(self, repository): git_dir = repository.git_dir branch = self.options.spinnaker_version logging.debug('Checking for branch="%s" in "%s"', branch, git_dir) remote_branches = [ line.strip() for line in self.__git.check_run(git_dir, 'branch -r').split('\n')] if 'origin/' + branch in remote_branches: if self.options.skip_existing: logging.info('Branch "%s" already exists in "%s" -- skip', branch, repository.origin) return elif self.options.delete_existing: logging.warning('Branch "%s" already exists in "%s" -- delete', branch, repository.origin) self.__git.delete_branch_on_origin(git_dir, branch) else: raise_and_log_error( ConfigError( 'Branch "{branch}" already exists in "{repo}"'.format( branch=branch, repo=repository.name), cause='branch_exists')) logging.info('Creating and pushing branch "%s" to "%s"', branch, repository.origin) self.__git.check_run(git_dir, 'checkout -b ' + branch) self.__git.push_branch_to_origin(git_dir, branch)
def __init__(self, factory, options, **kwargs): super(InitiateReleaseBranchCommand, self).__init__( factory, options, **kwargs) check_options_set(options, ['spinnaker_version']) self.__git = GitRunner(options)
class SpinnakerSourceCodeManager(object): """Helper class for managing spinnaker source code repositories.""" AUTO = '_auto_' @staticmethod def add_parser_args(parser, defaults): """Add standard parser arguments used by SourceCodeManager.""" if hasattr(parser, 'added_scm'): return parser.added_scm = True GitRunner.add_parser_args(parser, defaults) add_parser_argument(parser, 'github_upstream_owner', defaults, 'spinnaker', help='The standard upstream repository owner.') @property def git(self): return self.__git @property def options(self): return self.__options @property def root_source_dir(self): """The base directory for all the source repositories. Each repository will be a child directory of this path. """ return self.__root_source_dir def __init__(self, options, root_source_dir, **kwargs): self.__max_threads = kwargs.pop('max_threads', 100) self.__add_upstream = kwargs.pop('attach_upstream', False) check_kwargs_empty(kwargs) self.__options = options self.__git = GitRunner(options) self.__root_source_dir = root_source_dir def service_name_to_repository_name(self, service_name): if service_name == 'monitoring-daemon': return 'spinnaker-monitoring' return service_name def repository_name_to_service_name(self, repository_name): if repository_name == 'spinnaker-monitoring': return 'monitoring-daemon' return repository_name def check_repository_is_current(self, repository): raise NotImplementedError(self.__class__.__name__) def determine_build_number(self, repository): raise NotImplementedError(self.__class__.__name__) def determine_origin(self, name): """Determine the origin URL for the given repository name.""" raise NotImplementedError(self.__class__.__name__) def make_repository_spec(self, name, **kwargs): """Create GitRepositorySpec based on the name and configuration. Args: git_dir: if supplied then use it, otherwise default under the root path. origin: if supplied then use it, even if None. Otherwise default upstream: if supplied then use it, even if None. Otherwise default. kwargs: Additional repository attributes """ git_dir = kwargs.pop('git_dir', os.path.join(self.__root_source_dir, name)) origin = kwargs.pop('origin', self.AUTO) upstream = kwargs.pop('upstream', self.AUTO) if origin == self.AUTO: origin = self.determine_origin(name) if os.path.exists(git_dir): logging.info('Confirming existing %s matches expectations', git_dir) existing = self.__git.determine_git_repository_spec(git_dir) if existing.origin != origin: raise_and_log_error( UnexpectedError( 'Repository "{dir}" origin="{have}" expected="{want}"'.format( dir=git_dir, have=existing.origin, want=origin))) if upstream == self.AUTO: upstream = self.determine_upstream_url(name) return GitRepositorySpec( name, origin=origin, upstream=upstream, git_dir=git_dir, **kwargs) def determine_upstream_url(self, name): upstream_owner = (self.__options.github_upstream_owner if name not in ('citest') else 'google') return 'https://github.com/{upstream}/{name}'.format( upstream=upstream_owner, name=name) def ensure_git_path(self, repository, **kwargs): """Make sure the repository is checked out. Normally one would use ensure_repository which also ensures the metadata is cached. However repositories that are not version controlled in the normal way (e.g. spinnaker.github.io) dont use the metadata so the assumptions in ensure_repository are not applicable. Returns: The build number to use """ raise NotImplementedError(self.__class__.__name__) def ensure_local_repository(self, repository, commit=None): """Make sure local repository directory exists, and make it so if not.""" git_dir = repository.git_dir have_git_dir = os.path.exists(git_dir) if have_git_dir: self.check_repository_is_current(repository) else: self.ensure_git_path(repository) def refresh_source_info(self, repository, build_number): """Extract the source info from repository and cache with build number. We associate the build number because the different builds (debian, container, etc) need to have the same builder number so that the eventual BOM is consitent. Since we dont build everything at once, we'll need to remember it. We extract out the repository summary info, particularly the commit it is at, to ensure that future operations are consistent and operating on the same commit. """ summary = self.__git.collect_repository_summary(repository.git_dir) expect_build_number = (self.__options.build_number if hasattr(self.__options, 'build_number') else build_number) info = SourceInfo(expect_build_number, summary) filename = repository.name + '-meta.yml' dir_path = os.path.join(self.__options.output_dir, 'source_info') cache_path = os.path.join(dir_path, filename) logging.debug( 'Refreshing source info for %s and caching to %s for buildnum=%s', repository.name, cache_path, build_number) write_to_path(info.summary.to_yaml(), cache_path) return info def lookup_source_info(self, repository): """Return the SourceInfo for the given repository.""" filename = repository.name + '-meta.yml' dir_path = os.path.join(self.__options.output_dir, 'source_info') build_number = self.determine_build_number(repository) with open(os.path.join(dir_path, filename), 'r') as stream: return SourceInfo( build_number, RepositorySummary.from_dict(yaml.safe_load(stream.read()))) def check_source_info(self, repository): """Ensure cached source info is consistent with current repository.""" logging.debug('Checking that cached commit is consistent with %s', repository.git_dir) info = self.lookup_source_info(repository) commit = self.__git.query_local_repository_commit_id(repository.git_dir) cached_commit = info.summary.commit_id if cached_commit != commit: raise_and_log_error( UnexpectedError( 'Cached commit {cache} != current commit {id} in {dir}'.format( cache=cached_commit, id=commit, dir=repository.git_dir))) return info def foreach_source_repository( self, all_repos, call_function, *posargs, **kwargs): """Call the function on each of the SourceRepository instances.""" worker = RepositoryWorker(call_function, *posargs, **kwargs) num_threads = min(self.__max_threads, len(all_repos)) if num_threads > 1: pool = ThreadPool(num_threads) logging.info('Mapping %d/%s', len(all_repos), [repo.name for repo in all_repos]) try: raw_list = pool.map(worker, all_repos) result = {name: value for name, value in raw_list} except Exception: logging.error('Map caught exception') pool.close() pool.join() raise logging.info('Finished mapping') pool.close() pool.join() else: # If we have only one thread, skip the pool # this is primarily to make debugging easier. result = { repository.name: worker(repository)[1] for repository in all_repos } return result def push_to_origin_if_not_upstream(self, repository, branch): """Push the local repository back to the origin, but not upstream.""" git_dir = repository.git_dir origin = repository.origin upstream = repository.upstream_or_none() if upstream is None: logging.warning('Skipping push origin %s because upstream is None.', repository.name) return if origin == upstream: logging.warning('Skipping push origin %s because origin is upstream.', repository.name) return self.__git.push_branch_to_origin(git_dir, branch) def determine_source_repositories(self): """Determine which repositories are available to this SCM.""" raise_and_log_error( UnexpectedError(self.__class__.__name__ + ': Should only be applicable to BomSCM', cause='NotReachable'))
class PublishSpinnakerCommand(CommandProcessor): """"Implements the publish_spinnaker command.""" # pylint: disable=too-few-public-methods def __init__(self, factory, options, **kwargs): super(PublishSpinnakerCommand, self).__init__(factory, options, **kwargs) check_options_set(options, [ 'spinnaker_version', 'spinnaker_release_alias', 'bom_version', 'changelog_gist_url', 'github_owner', 'min_halyard_version' ]) major, minor, _ = self.options.spinnaker_version.split('.') self.__branch = 'release-{major}.{minor}.x'.format(major=major, minor=minor) options_copy = copy.copy(options) self.__scm = BomSourceCodeManager(options_copy, self.get_input_dir()) self.__hal = HalRunner(options) self.__git = GitRunner(options) self.__hal.check_property('spinnaker.config.input.bucket', options.halyard_bom_bucket) if options.only_repositories: self.__only_repositories = options.only_repositories.split(',') else: self.__only_repositories = [] options_copy.git_branch = self.__branch self.__process_scm = BranchSourceCodeManager(options_copy, self.get_input_dir()) def push_branches_and_tags(self, bom): """Update the release branches and tags in each of the BOM repositires.""" logging.info('Tagging each of the BOM service repos') # Run in two passes so we dont push anything if we hit a problem # in the tagging pass. Since we are spread against multiple repositiories, # we cannot do this atomically. The two passes gives us more protection # from a partial push due to errors in a repo. names_to_push = set([]) for which in ['tag', 'push']: for name, spec in bom['services'].items(): if name in ['monitoring-third-party', 'defaultArtifact']: # Ignore this, it is redundant to monitoring-daemon continue if name == 'monitoring-daemon': name = 'spinnaker-monitoring' if self.__only_repositories and name not in self.__only_repositories: logging.debug('Skipping %s because of --only_repositories', name) continue if spec is None: logging.warning('HAVE bom.services.%s = None', name) continue repository = self.__scm.make_repository_spec(name) self.__scm.ensure_local_repository(repository) if which == 'tag': added = self.__branch_and_tag_repository( repository, self.__branch) if added: names_to_push.add(name) else: self.__push_branch_and_maybe_tag_repository( repository, self.__branch, name in names_to_push) for name in SPINNAKER_PROCESS_REPOSITORY_NAMES: if self.__only_repositories and name not in self.__only_repositories: logging.debug('Skipping %s because of --only_repositories', name) continue repository = self.__process_scm.make_repository_spec(name) self.__process_scm.ensure_local_repository(repository) if self.__branch_and_tag_repository(repository, self.__branch): self.__push_branch_and_maybe_tag_repository( repository, self.__branch) def __already_have_tag(self, repository, tag): """Determine if we already have the tag in the repository.""" git_dir = repository.git_dir existing_commit = self.__git.query_commit_at_tag(git_dir, tag) if not existing_commit: return False want_commit = self.__git.query_local_repository_commit_id(git_dir) if want_commit == existing_commit: logging.debug('Already have "%s" at %s', tag, want_commit) return True raise_and_log_error( ConfigError( '"{tag}" already exists in "{repo}" at commit {have}, not {want}' .format(tag=tag, repo=git_dir, have=existing_commit, want=want_commit))) def __branch_and_tag_repository(self, repository, branch): """Create a branch and/or verison tag in the repository, if needed.""" version = self.__scm.determine_repository_version(repository) tag = 'version-' + version if self.__already_have_tag(repository, tag): return False self.__git.check_run(repository.git_dir, 'tag ' + tag) return True def __push_branch_and_maybe_tag_repository(self, repository, branch, also_tag): """Push the branch and verison tag to the origin.""" tag = 'version-' + self.__scm.determine_repository_version(repository) self.__git.push_branch_to_origin(repository.git_dir, branch) if also_tag: self.__git.push_tag_to_origin(repository.git_dir, tag) else: logging.info('%s was already tagged with "%s" -- skip', repository.git_dir, tag) def _do_command(self): """Implements CommandProcessor interface.""" options = self.options spinnaker_version = options.spinnaker_version options_copy = copy.copy(options) options_copy.git_branch = 'master' # push to master in spinnaker.github.io publish_changelog_command = PublishChangelogFactory().make_command( options_copy) changelog_gist_url = options.changelog_gist_url # Make sure changelog exists already. # If it does not then fail. try: logging.debug('Verifying changelog ready at %s', changelog_gist_url) urlopen(changelog_gist_url) except HTTPError as error: logging.error(exception_to_message) raise_and_log_error( ConfigError( 'Changelog gist "{url}" must exist before publising a release.' .format(url=changelog_gist_url), cause='ChangelogMissing')) bom = self.__hal.retrieve_bom_version(self.options.bom_version) bom['version'] = spinnaker_version bom_path = os.path.join(self.get_output_dir(), spinnaker_version + '.yml') write_to_path(yaml.safe_dump(bom, default_flow_style=False), bom_path) self.__hal.publish_bom_path(bom_path) self.push_branches_and_tags(bom) self.__hal.publish_spinnaker_release(spinnaker_version, options.spinnaker_release_alias, changelog_gist_url, options.min_halyard_version) logging.info('Publishing changelog') publish_changelog_command()
class PushChangelogCommand(CommandProcessor): """Implements push_changelog_to_gist.""" def __init__(self, factory, options, **kwargs): super(PushChangelogCommand, self).__init__(factory, options, **kwargs) check_options_set( options, ['build_changelog_gist_url', 'git_branch']) if not options.changelog_path: options.changelog_path = os.path.join( self.get_output_dir(command=BUILD_CHANGELOG_COMMAND), 'changelog.md') check_path_exists(options.changelog_path, why='changelog_path') self.__git = GitRunner(options) def _do_command(self): options = self.options gist_url = options.build_changelog_gist_url index = gist_url.rfind('/') if index < 0: index = gist_url.rfind(':') # ssh gist gist_id = gist_url[index + 1:] git_dir = os.path.join(self.get_input_dir(), gist_id) if not os.path.exists(git_dir): logging.debug('Cloning gist from %s', gist_url) ensure_dir_exists(os.path.dirname(git_dir)) self.__git.check_run(os.path.dirname(git_dir), 'clone ' + gist_url) else: logging.debug('Updating gist in "%s"', git_dir) self.__git.check_run(git_dir, 'fetch origin master') self.__git.check_run(git_dir, 'checkout master') dest_path = os.path.join( git_dir, '%s-raw-changelog.md' % options.git_branch) logging.debug('Copying "%s" to "%s"', options.changelog_path, dest_path) shutil.copyfile(options.changelog_path, dest_path) self.__git.check_run(git_dir, 'add ' + os.path.basename(dest_path)) self.__git.check_commit_or_no_changes( git_dir, '-a -m "Updated %s"' % os.path.basename(dest_path)) logging.debug('Pushing back gist') self.__git.check_run(git_dir, 'push -f origin master')
def __init__(self, factory, options, **kwargs): super(InitiateReleaseBranchCommand, self).__init__(factory, options, **kwargs) check_options_set(options, ['spinnaker_version']) self.__git = GitRunner(options)
def setUpClass(cls): cls.git = GitRunner(make_default_options()) cls.base_temp_dir = tempfile.mkdtemp(prefix='git_test') cls.git_dir = os.path.join(cls.base_temp_dir, 'commit_message_test') os.makedirs(cls.git_dir) git_dir = cls.git_dir gitify = lambda args: 'git -C "{dir}" {args}'.format(dir=git_dir, args=args) check_subprocess_sequence([ gitify('init'), 'touch "{dir}/base_file"'.format(dir=git_dir), gitify('add "{dir}/base_file"'.format(dir=git_dir)), gitify('commit -a -m "feat(test): added file"'), gitify( 'tag {base_version} HEAD'.format(base_version=VERSION_BASE)), # For testing patches gitify('checkout -b {patch_branch}'.format( patch_branch=cls.PATCH_BRANCH)), 'touch "{dir}/patch_file"'.format(dir=git_dir), gitify('add "{dir}/patch_file"'.format(dir=git_dir)), gitify('commit -a -m "fix(testA): added patch_file"'), # For testing minor versions gitify('checkout -b {minor_branch}'.format( minor_branch=cls.MINOR_BRANCH)), 'touch "{dir}/minor_file"'.format(dir=git_dir), gitify('add "{dir}/minor_file"'.format(dir=git_dir)), gitify('commit -a -m "feat(testB): added minor_file"'), # For testing major versions gitify('checkout -b {major_branch}'.format( major_branch=cls.MAJOR_BRANCH)), 'touch "{dir}/major_file"'.format(dir=git_dir), gitify('add "{dir}/major_file"'.format(dir=git_dir)), gitify('commit -a -m' ' "feat(testC): added major_file\n' '\nInterestingly enough, this is a BREAKING CHANGE.' '"'), # For testing composite commits from a merge of commits gitify('checkout -b {merged_branch}'.format( merged_branch=cls.MERGED_BRANCH)), gitify('reset --hard HEAD~3'), gitify('merge --squash HEAD@{1}'), ]) env = dict(os.environ) if os.path.exists('/bin/true'): env['EDITOR'] = '/bin/true' elif os.path.exists('/usr/bin/true'): env['EDITOR'] = '/usr/bin/true' else: raise NotImplementedError('platform not supported for this test') check_subprocess('git -C "{dir}" commit'.format(dir=git_dir), env=env) # For testing changelog from a commit check_subprocess_sequence([ gitify('checkout {minor_branch}'.format( minor_branch=cls.MINOR_BRANCH)), gitify('checkout -b {x_branch}'.format( x_branch=cls.PATCH_MINOR_BRANCH)), 'touch "{dir}/xbefore_file"'.format(dir=git_dir), gitify('add "{dir}/xbefore_file"'.format(dir=git_dir)), gitify('commit -a -m "feat(test): COMMIT AT TAG"'), gitify('tag {x_marker} HEAD'.format(x_marker=cls.PATCH_MINOR_X)), 'touch "{dir}/x_first"'.format(dir=git_dir), gitify('add "{dir}/x_first"'.format(dir=git_dir)), gitify('commit -a -m "fix(test): First Fix"'), 'rm "{dir}/x_first"'.format(dir=git_dir), gitify('commit -a -m "fix(test): Second Fix"'), ])
class GradleRunner(object): """Helper module for running gradle.""" __GRADLE_PUBLISH_FILE = os.path.join("gradle", "init-publish.gradle") @staticmethod def add_parser_args(parser, defaults): """Add parser arguments for gradle.""" if hasattr(parser, "added_gradle_runner"): return parser.added_gradle_runner = True add_parser_argument( parser, "bintray_jar_repository", defaults, None, help= "bintray repository in the bintray_org to publish jar files into.", ) add_parser_argument( parser, "gradle_cache_path", defaults, "{home}/.gradle".format( home=os.environ["HOME"]) if os.environ.get("HOME") else None, help="Path to a gradle cache directory to use for the builds.", ) add_parser_argument( parser, "gradle_network_timeout_secs", defaults, 60, help="Seconds to configure gradle timeouts (e.g. with bintray).", ) add_parser_argument( parser, "maven_custom_init_file", defaults, os.path.join(os.path.dirname(__file__), "..", "maven-init.gradle"), help="Path to a gradle init file to add to the debian builds." " Used to specify any custom behavior in the gradle builds." " Argument is a file path relative to the directory this script is" " executed in." " The default value assumes we run this script from the parent" " directory of spinnaker/buildtool.", ) @property def source_code_manager(self): """Return bound source code manager.""" return self.__scm def __init__(self, options, scm, metrics): self.__options = options self.__metrics = metrics self.__git = GitRunner(options) self.__scm = scm def __to_bintray_url(self, repo, package_name, repository, build_version): """Return the url for the desired versioned repository in bintray repo.""" bintray_path = "packages/{subject}/{repo}/{package}/versions/{version}".format( subject=self.__options.bintray_org, package=package_name, repo=repo, version=build_version, ) return "https://api.bintray.com/" + bintray_path def __add_bintray_auth_header(self, request): """Adds bintray authentication header to the request.""" user = os.environ["BINTRAY_USER"] password = os.environ["BINTRAY_KEY"] encoded_auth = base64.b64encode("{user}:{password}".format( user=user, password=password)) request.add_header("Authorization", "Basic " + bytes.decode(encoded_auth)) def bintray_repo_has_version(self, repo, package_name, repository, build_version): """See if the given bintray repository has the package version to build.""" try: bintray_url = self.__to_bintray_url(repo, package_name, repository, build_version) logging.debug("Checking for %s", bintray_url) request = Request(url=bintray_url) self.__add_bintray_auth_header(request) urlopen(request) return True except HTTPError as ex: if ex.code == 404: return False raise_and_log_error( ResponseError("Bintray failure: {}".format(ex), server="bintray.check"), "Failed on url=%s: %s" % (bintray_url, exception_to_message(ex)), ) except Exception as ex: raise def consider_debian_on_bintray(self, repository, build_version): """Check whether desired version already exists on bintray.""" options = self.__options exists = [] missing = [] # technically we publish to both maven and debian repos. # we can be in a state where we are in one but not the other. # let's not worry about this for now. for bintray_repo in [options.bintray_debian_repository]: # , # options.bintray_jar_repository]: package_name = repository.name if bintray_repo == options.bintray_debian_repository: if package_name == "spinnaker-monitoring": package_name = "spinnaker-monitoring-daemon" elif not package_name.startswith("spinnaker"): package_name = "spinnaker-" + package_name if self.bintray_repo_has_version(bintray_repo, package_name, repository, build_version): exists.append(bintray_repo) else: missing.append(bintray_repo) if exists: if options.skip_existing: if missing: raise_and_log_error( ConfigError( "Have {name} version for {exists} but not {missing}" .format( name=repository.name, exists=exists[0], missing=missing[0], ))) logging.info("Already have %s -- skipping build", repository.name) labels = {"repository": repository.name, "artifact": "debian"} self.__metrics.inc_counter("ReuseArtifact", labels) return True if options.delete_existing: for repo in exists: self.bintray_repo_delete_version( repo, package_name, repository, build_version=build_version) else: raise_and_log_error( ConfigError("Already have debian for {name}".format( name=repository.name))) return False def bintray_repo_delete_version(self, repo, package_name, repository, build_version=None): """Delete the given bintray repository version if it exsts.""" try: bintray_url = self.__to_bintray_url(repo, package_name, repository, build_version) logging.debug("Checking for %s", bintray_url) request = Request(url=bintray_url) request.get_method = lambda: "DELETE" self.__add_bintray_auth_header(request) labels = { "repo": repo, "repository": repository.name, "artifact": "debian" } self.__metrics.count_call("DeleteArtifact", labels, urlopen, request) return True except HTTPError as ex: if ex.code == 404: return True raise_and_log_error( ResponseError("Bintray failure: {}".format(ex), server="bintray.delete"), "Failed on url=%s: %s" % (bintray_url, exception_to_message(ex)), ) def get_common_args(self): """Return standard gradle args.""" options = self.__options args = [ "--stacktrace", "--info", ] if options.maven_custom_init_file: # Note, this was only debians args.append("-I {}".format(options.maven_custom_init_file)) return args def get_debian_args(self, distribution): """Return the debian args for the given distribution name.""" bintray_key = os.environ["BINTRAY_KEY"] bintray_user = os.environ["BINTRAY_USER"] options = self.__options bintray_org = options.bintray_org jar_repo = options.bintray_jar_repository debian_repo = options.bintray_debian_repository publish_wait_secs = options.bintray_publish_wait_secs args = [ '-PbintrayOrg="{org}"'.format(org=bintray_org), '-PbintrayPackageRepo="{repo}"'.format(repo=debian_repo), '-PbintrayJarRepo="{jarRepo}"'.format(jarRepo=jar_repo), '-PbintrayKey="{key}"'.format(key=bintray_key), '-PbintrayUser="******"'.format(user=bintray_user), "-PbintrayPackageDebDistribution={distribution}".format( distribution=distribution), "-PbintrayPublishWaitForSecs={publishWaitSecs}".format( publishWaitSecs=publish_wait_secs), ] return args def check_run( self, args, command_processor, repository, target, context, version, build_number, gradle_dir=None, ): """Run the gradle command on the given repository.""" gradle_dir = gradle_dir or repository.git_dir if self.__has_init_publish_file(repository): args.extend(["-I", self.__GRADLE_PUBLISH_FILE]) if self.__is_plugin_version_6(repository): args.extend(["-Pversion=%s-%s" % (version, build_number)]) else: args.append("-Prelease.useLastTag=true") build_number = self.prepare_local_git_for_nebula( gradle_dir, repository, version=version, build_number=build_number) full_args = list(args) full_args.append("-PbintrayPackageBuildNumber=%s" % build_number) # This gradle options wasnt introduced until 4.10.2 timeout = self.__options.gradle_network_timeout_secs * 1000 if timeout: full_args.append("-Dorg.gradle.internal.http.socketTimeout=%d" % timeout) full_args.append( "-Dorg.gradle.internal.http.connectionTimeout=%d" % timeout) name = repository.name logfile = command_processor.get_logfile_path(name + "-" + context) cmd = "./gradlew {args} {target}".format(args=" ".join(full_args), target=target) labels = { "repository": repository.name, "context": context, "target": target } self.__metrics.time_call( "GradleBuild", labels, self.__metrics.default_determine_outcome_labels, check_subprocesses_to_logfile, name + " gradle " + context, logfile, [cmd], cwd=gradle_dir, postprocess_hook=GradleMetricsUpdater(self.__metrics, repository, target), ) def __is_plugin_version_6(self, repository): return (not self.__has_init_publish_file(repository) and not repository.name == "spinnaker-monitoring") def __has_init_publish_file(self, repository): return os.path.isfile( os.path.join(repository.git_dir, self.__GRADLE_PUBLISH_FILE)) def prepare_local_git_for_nebula(self, gradle_dir, repository, version=None, build_number=None): """Tag the repository with the version we want to build. Args: version: optional version to tag with. If not provided then infer it. gradle_dir: The dir to prepare if supplied """ git_dir = gradle_dir or repository.git_dir self.__scm.ensure_local_repository(repository) self.__git.remove_all_non_version_tags(repository, git_dir=git_dir) if not build_number: build_number = self.__scm.determine_build_number(repository) # This doesn't really work because get_repository_service_build_version only # exists in BomSourceCodeManager. if not version: build_version = self.__scm.get_repository_service_build_version( repository) else: build_version = "%s-%s" % (version, build_number) logging.debug('Tagging repository %s with "%s" for nebula', git_dir, build_version) self.__git.tag_head(git_dir, build_version) return build_number
class GradleRunner(object): """Helper module for running gradle.""" @staticmethod def add_parser_args(parser, defaults): """Add parser arguments for gradle.""" if hasattr(parser, 'added_gradle_runner'): return parser.added_gradle_runner = True add_parser_argument( parser, 'bintray_jar_repository', defaults, None, help= 'bintray repository in the bintray_org to publish jar files into.') add_parser_argument( parser, 'gradle_cache_path', defaults, '{home}/.gradle'.format( home=os.environ['HOME']) if os.environ.get('HOME') else None, help='Path to a gradle cache directory to use for the builds.') add_parser_argument( parser, 'maven_custom_init_file', defaults, os.path.join(os.path.dirname(__file__), '..', 'maven-init.gradle'), help='Path to a gradle init file to add to the debian builds.' ' Used to specify any custom behavior in the gradle builds.' ' Argument is a file path relative to the directory this script is' ' executed in.' ' The default value assumes we run this script from the parent' ' directory of spinnaker/spinnaker.') @property def source_code_manager(self): """Return bound source code manager.""" return self.__scm def __init__(self, options, scm, metrics): self.__options = options self.__metrics = metrics self.__git = GitRunner(options) self.__scm = scm def __to_bintray_url(self, repo, package_name, repository, build_version=None, build_number=None): """Return the url for the desired versioned repository in bintray repo.""" if not build_version: source_info = self.__scm.lookup_source_info(repository) build_number = build_number or source_info.build_number build_version = '%s-%s' % (source_info.summary.version, build_number) bintray_path = ( 'packages/{subject}/{repo}/{package}/versions/{version}'.format( subject=self.__options.bintray_org, package=package_name, repo=repo, version=build_version)) return 'https://api.bintray.com/' + bintray_path def __add_bintray_auth_header(self, request): """Adds bintray authentication header to the request.""" user = os.environ['BINTRAY_USER'] password = os.environ['BINTRAY_KEY'] encoded_auth = base64.encodestring('{user}:{password}'.format( user=user, password=password))[:-1] # strip eoln request.add_header('Authorization', 'Basic ' + encoded_auth) def bintray_repo_has_version(self, repo, package_name, repository, build_version=None, build_number=None): """See if the given bintray repository has the package version to build.""" try: bintray_url = self.__to_bintray_url(repo, package_name, repository, build_version=build_version, build_number=build_number) logging.debug('Checking for %s', bintray_url) request = urllib2.Request(url=bintray_url) self.__add_bintray_auth_header(request) urllib2.urlopen(request) return True except urllib2.HTTPError as ex: if ex.code == 404: return False raise_and_log_error( ResponseError('Bintray failure: {}'.format(ex), server='bintray.check'), 'Failed on url=%s: %s' % (bintray_url, ex.message)) except Exception as ex: raise def consider_debian_on_bintray(self, repository, build_version=None, build_number=None): """Check whether desired version already exists on bintray.""" options = self.__options exists = [] missing = [] # technically we publish to both maven and debian repos. # we can be in a state where we are in one but not the other. # let's not worry about this for now. for bintray_repo in [options.bintray_debian_repository]: #, # options.bintray_jar_repository]: package_name = repository.name if bintray_repo == options.bintray_debian_repository: if package_name == 'spinnaker-monitoring': package_name = 'spinnaker-monitoring-daemon' elif not package_name.startswith('spinnaker'): package_name = 'spinnaker-' + package_name if self.bintray_repo_has_version(bintray_repo, package_name, repository, build_version=build_version, build_number=build_number): exists.append(bintray_repo) else: missing.append(bintray_repo) if exists: if options.skip_existing: if missing: raise_and_log_error( ConfigError( 'Have {name} version for {exists} but not {missing}' .format(name=repository.name, exists=exists[0], missing=missing[0]))) logging.info('Already have %s -- skipping build', repository.name) labels = {'repository': repository.name, 'artifact': 'debian'} self.__metrics.inc_counter( 'ReuseArtifact', labels, 'Kept existing desired debian package version.') return True if options.delete_existing: for repo in exists: self.bintray_repo_delete_version( repo, package_name, repository, build_version=build_version) else: raise_and_log_error( ConfigError('Already have debian for {name}'.format( name=repository.name))) return False def bintray_repo_delete_version(self, repo, package_name, repository, build_version=None): """Delete the given bintray repository version if it exsts.""" try: bintray_url = self.__to_bintray_url(repo, package_name, repository, build_version=build_version) logging.debug('Checking for %s', bintray_url) request = urllib2.Request(url=bintray_url) request.get_method = lambda: 'DELETE' self.__add_bintray_auth_header(request) labels = { 'repo': repo, 'repository': repository.name, 'artifact': 'debian' } self.__metrics.count_call( 'DeleteArtifact', labels, 'Attempts to delete versioned artifacts on bintray', urllib2.urlopen, request) return True except urllib2.HTTPError as ex: if ex.code == 404: return True raise_and_log_error( ResponseError('Bintray failure: {}'.format(ex), server='bintray.delete'), 'Failed on url=%s: %s' % (bintray_url, ex.message)) def get_common_args(self): """Return standard gradle args.""" options = self.__options args = [ '--stacktrace', '--info', '-Prelease.useLastTag=true', ] if options.maven_custom_init_file: # Note, this was only debians, not for rpms before. args.append('-I {}'.format(options.maven_custom_init_file)) return args def get_debian_args(self, distribution): """Return the debian args for the given distribution name.""" bintray_key = os.environ['BINTRAY_KEY'] bintray_user = os.environ['BINTRAY_USER'] options = self.__options bintray_org = options.bintray_org jar_repo = options.bintray_jar_repository debian_repo = options.bintray_debian_repository args = [ '-PbintrayOrg="{org}"'.format(org=bintray_org), '-PbintrayPackageRepo="{repo}"'.format(repo=debian_repo), '-PbintrayJarRepo="{jarRepo}"'.format(jarRepo=jar_repo), '-PbintrayKey="{key}"'.format(key=bintray_key), '-PbintrayUser="******"'.format(user=bintray_user), '-PbintrayPackageDebDistribution={distribution}'.format( distribution=distribution) ] return args def check_run(self, args, command_processor, repository, target, context, gradle_dir=None, **kwargs): """Run the gradle command on the given repository.""" gradle_dir = gradle_dir or repository.git_dir version = kwargs.pop('version', None) build_number = kwargs.pop('build_number', None) build_number = self.prepare_local_git_for_nebula( gradle_dir, repository, version=version, build_number=build_number) full_args = list(args) full_args.append('-PbintrayPackageBuildNumber=%s' % build_number) name = repository.name logfile = command_processor.get_logfile_path(name + '-' + context) cmd = './gradlew {args} {target}'.format(args=' '.join(full_args), target=target) labels = { 'repository': repository.name, 'context': context, 'target': target } self.__metrics.time_call('GradleBuild', labels, 'Gradle builds.', check_subprocesses_to_logfile, name + ' gradle ' + context, logfile, [cmd], cwd=gradle_dir, postprocess_hook=GradleMetricsUpdater( self.__metrics, repository, target)) def prepare_local_git_for_nebula(self, gradle_dir, repository, version=None, build_number=None): """Tag the repository with the version we want to build. Args: version: optional version to tag with. If not provided then infer it. gradle_dir: The dir to prepare if supplied """ git_dir = gradle_dir or repository.git_dir self.__scm.ensure_local_repository(repository) self.__git.remove_all_non_version_tags(repository, git_dir=git_dir) if not version or not build_number: source_info = self.__scm.lookup_source_info(repository) if not build_number: build_number = source_info.build_number if not version: version = source_info.summary.version build_version = '%s-%s' % (version, build_number) logging.debug('Tagging repository %s with "%s" for nebula', git_dir, build_version) self.__git.tag_head(git_dir, build_version) return build_number
def __init__(self, options, scm, metrics): self.__options = options self.__metrics = metrics self.__git = GitRunner(options) self.__scm = scm
class GradleRunner(object): """Helper module for running gradle.""" __GRADLE_PUBLISH_FILE = os.path.join("gradle", "init-publish.gradle") @staticmethod def add_parser_args(parser, defaults): """Add parser arguments for gradle.""" if hasattr(parser, 'added_gradle_runner'): return parser.added_gradle_runner = True add_parser_argument( parser, 'bintray_jar_repository', defaults, None, help='bintray repository in the bintray_org to publish jar files into.') add_parser_argument( parser, 'gradle_cache_path', defaults, '{home}/.gradle'.format(home=os.environ['HOME']) if os.environ.get('HOME') else None, help='Path to a gradle cache directory to use for the builds.') add_parser_argument( parser, 'gradle_network_timeout_secs', defaults, 60, help='Seconds to configure gradle timeouts (e.g. with bintray).') add_parser_argument( parser, 'maven_custom_init_file', defaults, os.path.join(os.path.dirname(__file__), '..', 'maven-init.gradle'), help='Path to a gradle init file to add to the debian builds.' ' Used to specify any custom behavior in the gradle builds.' ' Argument is a file path relative to the directory this script is' ' executed in.' ' The default value assumes we run this script from the parent' ' directory of spinnaker/spinnaker.') @property def source_code_manager(self): """Return bound source code manager.""" return self.__scm def __init__(self, options, scm, metrics): self.__options = options self.__metrics = metrics self.__git = GitRunner(options) self.__scm = scm def __to_bintray_url(self, repo, package_name, repository, build_version): """Return the url for the desired versioned repository in bintray repo.""" bintray_path = ( 'packages/{subject}/{repo}/{package}/versions/{version}'.format( subject=self.__options.bintray_org, package=package_name, repo=repo, version=build_version)) return 'https://api.bintray.com/' + bintray_path def __add_bintray_auth_header(self, request): """Adds bintray authentication header to the request.""" user = os.environ['BINTRAY_USER'] password = os.environ['BINTRAY_KEY'] encoded_auth = base64.encodestring(str.encode('{user}:{password}'.format( user=user, password=password)))[:-1] # strip eoln request.add_header('Authorization', 'Basic ' + bytes.decode(encoded_auth)) def bintray_repo_has_version(self, repo, package_name, repository, build_version): """See if the given bintray repository has the package version to build.""" try: bintray_url = self.__to_bintray_url(repo, package_name, repository, build_version) logging.debug('Checking for %s', bintray_url) request = Request(url=bintray_url) self.__add_bintray_auth_header(request) urlopen(request) return True except HTTPError as ex: if ex.code == 404: return False raise_and_log_error( ResponseError('Bintray failure: {}'.format(ex), server='bintray.check'), 'Failed on url=%s: %s' % (bintray_url, exception_to_message(ex))) except Exception as ex: raise def consider_debian_on_bintray(self, repository, build_version): """Check whether desired version already exists on bintray.""" options = self.__options exists = [] missing = [] # technically we publish to both maven and debian repos. # we can be in a state where we are in one but not the other. # let's not worry about this for now. for bintray_repo in [options.bintray_debian_repository]:#, # options.bintray_jar_repository]: package_name = repository.name if bintray_repo == options.bintray_debian_repository: if package_name == 'spinnaker-monitoring': package_name = 'spinnaker-monitoring-daemon' elif not package_name.startswith('spinnaker'): package_name = 'spinnaker-' + package_name if self.bintray_repo_has_version( bintray_repo, package_name, repository, build_version): exists.append(bintray_repo) else: missing.append(bintray_repo) if exists: if options.skip_existing: if missing: raise_and_log_error( ConfigError('Have {name} version for {exists} but not {missing}' .format(name=repository.name, exists=exists[0], missing=missing[0]))) logging.info('Already have %s -- skipping build', repository.name) labels = {'repository': repository.name, 'artifact': 'debian'} self.__metrics.inc_counter('ReuseArtifact', labels) return True if options.delete_existing: for repo in exists: self.bintray_repo_delete_version(repo, package_name, repository, build_version=build_version) else: raise_and_log_error( ConfigError('Already have debian for {name}'.format( name=repository.name))) return False def bintray_repo_delete_version(self, repo, package_name, repository, build_version=None): """Delete the given bintray repository version if it exsts.""" try: bintray_url = self.__to_bintray_url(repo, package_name, repository, build_version) logging.debug('Checking for %s', bintray_url) request = Request(url=bintray_url) request.get_method = lambda: 'DELETE' self.__add_bintray_auth_header(request) labels = { 'repo': repo, 'repository': repository.name, 'artifact': 'debian' } self.__metrics.count_call( 'DeleteArtifact', labels, urlopen, request) return True except HTTPError as ex: if ex.code == 404: return True raise_and_log_error( ResponseError('Bintray failure: {}'.format(ex), server='bintray.delete'), 'Failed on url=%s: %s' % (bintray_url, exception_to_message(ex))) def get_common_args(self): """Return standard gradle args.""" options = self.__options args = [ '--stacktrace', '--info', ] if options.maven_custom_init_file: # Note, this was only debians, not for rpms before. args.append('-I {}'.format(options.maven_custom_init_file)) return args def get_debian_args(self, distribution): """Return the debian args for the given distribution name.""" bintray_key = os.environ['BINTRAY_KEY'] bintray_user = os.environ['BINTRAY_USER'] options = self.__options bintray_org = options.bintray_org jar_repo = options.bintray_jar_repository debian_repo = options.bintray_debian_repository publish_wait_secs = options.bintray_publish_wait_secs args = [ '-PbintrayOrg="{org}"'.format(org=bintray_org), '-PbintrayPackageRepo="{repo}"'.format(repo=debian_repo), '-PbintrayJarRepo="{jarRepo}"'.format(jarRepo=jar_repo), '-PbintrayKey="{key}"'.format(key=bintray_key), '-PbintrayUser="******"'.format(user=bintray_user), '-PbintrayPackageDebDistribution={distribution}'.format( distribution=distribution), '-PbintrayPublishWaitForSecs={publishWaitSecs}'.format( publishWaitSecs=publish_wait_secs) ] return args def check_run(self, args, command_processor, repository, target, context, version, build_number, gradle_dir=None): """Run the gradle command on the given repository.""" gradle_dir = gradle_dir or repository.git_dir if self.__has_init_publish_file(repository): args.extend(['-I', self.__GRADLE_PUBLISH_FILE]) if self.__is_plugin_version_6(repository): args.extend(['-PenablePublishing=true', '-Prelease.disableGitChecks=true', '-Prelease.version=%s-%s' % (version, build_number)]) else: args.append('-Prelease.useLastTag=true') build_number = self.prepare_local_git_for_nebula( gradle_dir, repository, version=version, build_number=build_number) full_args = list(args) full_args.append('-PbintrayPackageBuildNumber=%s' % build_number) # This gradle options wasnt introduced until 4.10.2 timeout = self.__options.gradle_network_timeout_secs * 1000 if timeout: full_args.append('-Dorg.gradle.internal.http.socketTimeout=%d' % timeout) full_args.append('-Dorg.gradle.internal.http.connectionTimeout=%d' % timeout) name = repository.name logfile = command_processor.get_logfile_path(name + '-' + context) cmd = './gradlew {args} {target}'.format( args=' '.join(full_args), target=target) labels = { 'repository': repository.name, 'context': context, 'target': target } self.__metrics.time_call( 'GradleBuild', labels, self.__metrics.default_determine_outcome_labels, check_subprocesses_to_logfile, name + ' gradle ' + context, logfile, [cmd], cwd=gradle_dir, postprocess_hook=GradleMetricsUpdater(self.__metrics, repository, target)) def __is_plugin_version_6(self, repository): return not self.__has_init_publish_file( repository) and not repository.name == 'spinnaker-monitoring' def __has_init_publish_file(self, repository): return os.path.isfile( os.path.join(repository.git_dir, self.__GRADLE_PUBLISH_FILE)) def prepare_local_git_for_nebula( self, gradle_dir, repository, version=None, build_number=None): """Tag the repository with the version we want to build. Args: version: optional version to tag with. If not provided then infer it. gradle_dir: The dir to prepare if supplied """ git_dir = gradle_dir or repository.git_dir self.__scm.ensure_local_repository(repository) self.__git.remove_all_non_version_tags(repository, git_dir=git_dir) if not build_number: build_number = self.__scm.determine_build_number(repository) # This doesn't really work because get_repository_service_build_version only # exists in BomSourceCodeManager. if not version: build_version = self.__scm.get_repository_service_build_version( repository) else: build_version = '%s-%s' % (version, build_number) logging.debug('Tagging repository %s with "%s" for nebula', git_dir, build_version) self.__git.tag_head(git_dir, build_version) return build_number
class PublishSpinnakerCommand(CommandProcessor): """"Implements the publish_spinnaker command.""" # pylint: disable=too-few-public-methods def __init__(self, factory, options, **kwargs): super(PublishSpinnakerCommand, self).__init__(factory, options, **kwargs) check_options_set(options, [ 'spinnaker_version', 'spinnaker_release_alias', 'bom_version', 'changelog_gist_url', 'github_owner', 'min_halyard_version' ]) major, minor, _ = self.options.spinnaker_version.split('.') self.__branch = 'release-{major}.{minor}.x'.format( major=major, minor=minor) options_copy = copy.copy(options) self.__bom_scm = BomSourceCodeManager(options_copy, self.get_input_dir()) self.__hal = HalRunner(options) self.__git = GitRunner(options) self.__hal.check_property( 'spinnaker.config.input.bucket', options.halyard_bom_bucket) if options.only_repositories: self.__only_repositories = options.only_repositories.split(',') else: self.__only_repositories = [] options_copy.git_branch = self.__branch self.__branch_scm = BranchSourceCodeManager( options_copy, self.get_input_dir()) def push_branches_and_tags(self, bom): """Update the release branches and tags in each of the BOM repositires.""" logging.info('Tagging each of the BOM service repos') bom_scm = self.__bom_scm branch_scm = self.__branch_scm # Run in two passes so we dont push anything if we hit a problem # in the tagging pass. Since we are spread against multiple repositiories, # we cannot do this atomically. The two passes gives us more protection # from a partial push due to errors in a repo. names_to_push = set([]) for which in ['tag', 'push']: for name, spec in bom['services'].items(): if name in ['monitoring-third-party', 'defaultArtifact']: # Ignore this, it is redundant to monitoring-daemon continue if name == 'monitoring-daemon': name = 'spinnaker-monitoring' if self.__only_repositories and name not in self.__only_repositories: logging.debug('Skipping %s because of --only_repositories', name) continue if spec is None: logging.warning('HAVE bom.services.%s = None', name) continue repository = bom_scm.make_repository_spec(name) bom_scm.ensure_local_repository(repository) version = bom_scm.determine_repository_version(repository) if which == 'tag': added = self.__branch_and_tag_repository( repository, self.__branch, version) if added: names_to_push.add(name) else: self.__push_branch_and_maybe_tag_repository( repository, self.__branch, version, name in names_to_push) additional_repositories = list(SPINNAKER_PROCESS_REPOSITORY_NAMES) for name in additional_repositories: if self.__only_repositories and name not in self.__only_repositories: logging.debug('Skipping %s because of --only_repositories', name) continue repository = branch_scm.make_repository_spec(name) branch_scm.ensure_local_repository(repository) git_summary = self.__git.collect_repository_summary(repository.git_dir) version = git_summary.version if self.__branch_and_tag_repository( repository, self.__branch, version): self.__push_branch_and_maybe_tag_repository( repository, self.__branch, version, True) def __already_have_tag(self, repository, tag): """Determine if we already have the tag in the repository.""" git_dir = repository.git_dir existing_commit = self.__git.query_commit_at_tag(git_dir, tag) if not existing_commit: return False want_commit = self.__git.query_local_repository_commit_id(git_dir) if want_commit == existing_commit: logging.debug('Already have "%s" at %s', tag, want_commit) return True raise_and_log_error( ConfigError( '"{tag}" already exists in "{repo}" at commit {have}, not {want}' .format(tag=tag, repo=git_dir, have=existing_commit, want=want_commit))) return False # not reached def __branch_and_tag_repository(self, repository, branch, version): """Create a branch and/or version tag in the repository, if needed.""" tag = 'version-' + version if self.__already_have_tag(repository, tag): return False self.__git.check_run(repository.git_dir, 'tag ' + tag) return True def __push_branch_and_maybe_tag_repository(self, repository, branch, version, also_tag): """Push the branch and version tag to the origin.""" tag = 'version-' + version self.__git.push_branch_to_origin(repository.git_dir, branch) if also_tag: self.__git.push_tag_to_origin(repository.git_dir, tag) else: logging.info('%s was already tagged with "%s" -- skip', repository.git_dir, tag) def _do_command(self): """Implements CommandProcessor interface.""" options = self.options spinnaker_version = options.spinnaker_version options_copy = copy.copy(options) options_copy.git_branch = 'master' # push to master in spinnaker.github.io publish_changelog_command = PublishChangelogFactory().make_command( options_copy) changelog_gist_url = options.changelog_gist_url # Make sure changelog exists already. # If it does not then fail. try: logging.debug('Verifying changelog ready at %s', changelog_gist_url) urlopen(changelog_gist_url) except HTTPError: logging.error(exception_to_message) raise_and_log_error( ConfigError( 'Changelog gist "{url}" must exist before publising a release.' .format(url=changelog_gist_url), cause='ChangelogMissing')) bom = self.__hal.retrieve_bom_version(self.options.bom_version) bom['version'] = spinnaker_version bom_path = os.path.join(self.get_output_dir(), spinnaker_version + '.yml') write_to_path(yaml.safe_dump(bom, default_flow_style=False), bom_path) self.__hal.publish_bom_path(bom_path) self.push_branches_and_tags(bom) self.__hal.publish_spinnaker_release( spinnaker_version, options.spinnaker_release_alias, changelog_gist_url, options.min_halyard_version) logging.info('Publishing changelog') publish_changelog_command()
def init_argparser(self, parser, defaults): super(PublishHalyardCommandFactory, self).init_argparser(parser, defaults) GradleCommandFactory.add_bom_parser_args(parser, defaults) SpinnakerSourceCodeManager.add_parser_args(parser, defaults) GradleRunner.add_parser_args(parser, defaults) GitRunner.add_publishing_parser_args(parser, defaults) HalRunner.add_parser_args(parser, defaults) self.add_argument( parser, 'build_number', defaults, DEFAULT_BUILD_NUMBER, help= 'Publishing halyard requires a rebuild. This is the build number' ' to use when rebuilding halyard.') self.add_argument( parser, 'halyard_version', defaults, None, required=True, help='The semantic version of the release to publish.') self.add_argument( parser, 'halyard_version_commits_url', defaults, None, help='URL to file containing version and git commit for successful' ' nightly builds. By default this will be' ' "{filename}" in the' ' --halyard_bucket_base_url.'.format( filename=BuildHalyardCommand.HALYARD_VERSIONS_BASENAME)) self.add_argument( parser, 'halyard_bucket_base_url', defaults, None, help='Base Google Cloud Storage URL for writing halyard builds.') self.add_argument(parser, 'docs_repo_owner', defaults, None, help='Owner of the docs repo if one was' ' specified. The default is --github_owner.') self.add_argument( parser, 'skip_existing', defaults, False, type=bool, help='Skip builds if the desired version already exists on bintray.' ) self.add_argument( parser, 'delete_existing', defaults, None, type=bool, help='Delete pre-existing desired versions if from bintray.')
class PushChangelogCommand(CommandProcessor): """Implements push_changelog_to_gist.""" def __init__(self, factory, options, **kwargs): super(PushChangelogCommand, self).__init__(factory, options, **kwargs) check_options_set(options, ['build_changelog_gist_url', 'git_branch']) if not options.changelog_path: options.changelog_path = os.path.join( self.get_output_dir(command=BUILD_CHANGELOG_COMMAND), 'changelog.md') check_path_exists(options.changelog_path, why='changelog_path') self.__git = GitRunner(options) def _do_command(self): options = self.options gist_url = options.build_changelog_gist_url index = gist_url.rfind('/') if index < 0: index = gist_url.rfind(':') # ssh gist gist_id = gist_url[index + 1:] git_dir = os.path.join(self.get_input_dir(), gist_id) if not os.path.exists(git_dir): logging.debug('Cloning gist from %s', gist_url) ensure_dir_exists(os.path.dirname(git_dir)) self.__git.check_run(os.path.dirname(git_dir), 'clone ' + gist_url) else: logging.debug('Updating gist in "%s"', git_dir) self.__git.check_run(git_dir, 'fetch origin master') self.__git.check_run(git_dir, 'checkout master') dest_path = os.path.join(git_dir, '%s-raw-changelog.md' % options.git_branch) logging.debug('Copying "%s" to "%s"', options.changelog_path, dest_path) shutil.copyfile(options.changelog_path, dest_path) self.__git.check_run(git_dir, 'add ' + os.path.basename(dest_path)) self.__git.check_commit_or_no_changes( git_dir, '-a -m "Updated %s"' % os.path.basename(dest_path)) logging.debug('Pushing back gist') self.__git.check_run(git_dir, 'push -f origin master')
def make_test_options(self): options = super(TestBomSourceCodeManager, self).make_test_options() parser = argparse.ArgumentParser() parser.add_argument('--output_dir', default=options.output_dir) GitRunner.add_parser_args(parser, {'github_owner': 'test_github_owner'}) return parser.parse_args()