def init_argparser(self, parser, defaults): """Implements CommandFactory interface.""" # For the spinnaker.github.io repository BranchSourceCodeManager.add_parser_args(parser, defaults) self.add_argument( parser, 'generate_swagger_path', defaults, 'swagger/generate_swagger.sh', help='Path to standard script (relative to repository) for' ' generating swagger JSON document. If this is empty then' ' spawn a live server to query instead. It is assumed that' ' if present, this will write a "swagger.json" into' ' its directory.') self.add_argument( parser, 'max_wait_secs_startup', defaults, 210, type=int, help='Number of seconds to wait for gate server to startup' ' before giving up and timing out.' ' Used only if generate_swagger_path is not empty.') self.add_argument( parser, 'swagger_codegen_cli_jar_path', defaults, None, help='The location of the swagger-codegen-cli jarfile.' ' The file can be obtained from' ' http://central.maven.org/maven2/io/swagger/swagger-codegen-cli' '/2.2.3/swagger-codegen-cli-2.2.3.jar') super(BuildApiDocsFactory, self).init_argparser(parser, defaults)
def __init__(self, factory, options, **kwargs): super(PublishSpinnakerCommand, self).__init__(factory, options, **kwargs) check_options_set(options, [ 'spinnaker_version', 'spinnaker_release_alias', 'bom_version', 'changelog_gist_url', 'github_owner', 'min_halyard_version' ]) major, minor, _ = self.options.spinnaker_version.split('.') self.__branch = 'release-{major}.{minor}.x'.format( major=major, minor=minor) options_copy = copy.copy(options) self.__scm = BomSourceCodeManager(options_copy, self.get_input_dir()) self.__hal = HalRunner(options) self.__git = GitRunner(options) self.__hal.check_property( 'spinnaker.config.input.bucket', options.halyard_bom_bucket) if options.only_repositories: self.__only_repositories = options.only_repositories.split(',') else: self.__only_repositories = [] options_copy.git_branch = self.__branch self.__process_scm = BranchSourceCodeManager( options_copy, self.get_input_dir())
def __init__(self, factory, options, **kwargs): options_copy = copy.copy(options) options_copy.bom_path = None options_copy.bom_version = None options_copy.git_branch = 'master' options_copy.github_hostname = 'github.com' # Overrides later if --git_allow_publish_master_branch is false super(PublishHalyardCommand, self).__init__(factory, options_copy, **kwargs) check_options_set(options, ['halyard_version']) match = re.match(r'(\d+)\.(\d+)\.(\d+)-\d+', options.halyard_version) if match is None: raise_and_log_error( ConfigError('--halyard_version={version} is not X.Y.Z-<buildnum>' .format(version=options.halyard_version))) self.__stable_version = '{major}.{minor}.{patch}'.format( major=match.group(1), minor=match.group(2), patch=match.group(3)) self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir()) self.__hal = HalRunner(options_copy) self.__gradle = GradleRunner(options_copy, self.__scm, self.metrics) self.__halyard_repo_md_path = os.path.join('docs', 'commands.md') dash = self.options.halyard_version.find('-') semver_str = self.options.halyard_version[0:dash] semver_parts = semver_str.split('.') if len(semver_parts) != 3: raise_and_log_error( ConfigError('Expected --halyard_version in the form X.Y.Z-N')) self.__release_branch = 'release-{maj}.{min}.x'.format( maj=semver_parts[0], min=semver_parts[1]) self.__release_tag = 'version-' + semver_str self.__release_version = semver_str
def test_maybe_pull_repository_branch(self): test_root = os.path.join(self.base_temp_dir, 'pulled_test') options = make_default_options() options.git_branch = UNTAGGED_BRANCH options.build_number = 'maybe_pull_branch_buildnum' scm = BranchSourceCodeManager(options, test_root) for repository_name, origin in self.ORIGIN_URLS.items(): repository = scm.make_repository_spec(repository_name, origin=origin, upstream=None) scm.ensure_local_repository(repository) git_dir = repository.git_dir spec = scm.git.determine_git_repository_spec(git_dir) self.assertEqual(repository, spec) in_branch = scm.git.query_local_repository_branch(git_dir) self.assertEqual(UNTAGGED_BRANCH, in_branch) summary = scm.git.collect_repository_summary(git_dir) semver = SemanticVersion.make(BASE_VERSION) expect_version = semver.next( SemanticVersion.MINOR_INDEX).to_version() self.assertEqual(expect_version, summary.version)
def init_argparser(self, parser, defaults): super(BuildContainerFactory, self).init_argparser(parser, defaults) self.add_bom_parser_args(parser, defaults) BranchSourceCodeManager.add_parser_args(parser, defaults) self.add_argument( parser, 'gcb_project', defaults, None, help='The GCP project ID that builds the containers when' ' using Google Container Builder.') self.add_argument( parser, 'gcb_service_account', defaults, None, help='Google Service Account when using the GCP Container Builder.' ) self.add_argument( parser, 'gcb_cache_bucket', defaults, "spinnaker-build-cache", help= 'Google Storage Bucket for build caches when using the GCP Container Builder.' )
def init_argparser(self, parser, defaults): """Implements CommandFactory interface.""" # For the spinnaker.github.io repository BranchSourceCodeManager.add_parser_args(parser, defaults) self.add_argument( parser, 'generate_swagger_path', defaults, 'swagger/generate_swagger.sh', help='Path to standard script (relative to repository) for' ' generating swagger JSON document. If this is empty then' ' spawn a live server to query instead. It is assumed that' ' if present, this will write a "swagger.json" into' ' its directory.') self.add_argument( parser, 'max_wait_secs_startup', defaults, 210, type=int, help='Number of seconds to wait for gate server to startup' ' before giving up and timing out.' ' Used only if generate_swagger_path is not empty.') self.add_argument( parser, 'swagger_codegen_cli_jar_path', defaults, None, help='The location of the swagger-codegen-cli jarfile.' ' The file can be obtained from' ' http://central.maven.org/maven2/io/swagger/swagger-codegen-cli' '/2.2.3/swagger-codegen-cli-2.2.3.jar') super(BuildApiDocsFactory, self).init_argparser(parser, defaults)
def __init__(self, factory, options, **kwargs): options_copy = copy.copy(options) options_copy.bom_path = None options_copy.bom_version = None options_copy.git_branch = 'master' # Overrides later if --git_allow_publish_master_branch is false super(PublishHalyardCommand, self).__init__(factory, options_copy, **kwargs) self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir()) self.__hal = HalRunner(options_copy) self.__gradle = GradleRunner(options_copy, self.__scm, self.metrics) self.__halyard_repo_md_path = os.path.join('docs', 'commands.md') dash = self.options.halyard_version.find('-') semver_str = self.options.halyard_version[0:dash] semver_parts = semver_str.split('.') if len(semver_parts) != 3: raise_and_log_error( ConfigError('Expected --halyard_version in the form X.Y.Z-N')) self.__release_branch = 'release-{maj}.{min}.x'.format( maj=semver_parts[0], min=semver_parts[1]) self.__release_tag = 'version-' + semver_str self.__release_version = semver_str
def test_get_local_repository_path(self): test_root = os.path.join(self.base_temp_dir, 'unused_source') scm = BranchSourceCodeManager(self.options, test_root) tests = ['RepoOne', 'RepoTwo', 'RepoTest', 'DoesNotExist'] for repo_name in tests: repository = scm.make_repository_spec(repo_name) expect = os.path.join(test_root, repo_name) self.assertEqual(expect, repository.git_dir) self.assertFalse(os.path.exists(expect))
def init_argparser(self, parser, defaults): super(BuildDebianFactory, self).init_argparser(parser, defaults) self.add_bom_parser_args(parser, defaults) BranchSourceCodeManager.add_parser_args(parser, defaults) self.add_argument( parser, 'gcb_project', defaults, None, help='The GCP project ID when using the GCP Container Builder.') self.add_argument( parser, 'gcb_service_account', defaults, None, help='Google Service Account when using the GCP Container Builder.')
def __init__(self, factory, options, **kwargs): super(PublishApiDocsCommand, self).__init__(factory, options, **kwargs) docs_dir = self.get_output_dir(command=BUILD_DOCS_COMMAND) self.__html_path = os.path.join(os.path.abspath(docs_dir), 'index.html') self._check_args() self.__scm = BranchSourceCodeManager( make_options_with_fallback(self.options), self.get_input_dir()) self.__docs_repository = self.__scm.make_repository_spec( SPINNAKER_GITHUB_IO_REPOSITORY_NAME)
def _ensure_templates_directory(self): """Initialize the templates_directory, pulling the repo if needed.""" scm = BranchSourceCodeManager( make_options_with_fallback(self.options), self.get_input_dir() ) repository = scm.make_repository_spec(SPINNAKER_IO_REPOSITORY_NAME) # These documents arent tied to version control, especially since they are # published to a different repository. scm.ensure_git_path(repository) self.__templates_directory = os.path.join(repository.git_dir, "_api_templates")
def test_build(self): test_root = self.test_root options = self.options options.git_branch = PATCH_BRANCH options.github_owner = 'default' options.github_disable_upstream_push = True scm = BranchSourceCodeManager(options, test_root) golden_bom = dict(self.golden_bom) builder = BomBuilder.new_from_bom(options, scm, golden_bom) source_repositories = [ scm.make_repository_spec(name) for name in ALL_STANDARD_TEST_BOM_REPO_NAMES ] for repository in source_repositories: scm.ensure_git_path(repository) summary = scm.git.collect_repository_summary(repository.git_dir) source_info = SourceInfo('SourceInfoBuildNumber', summary) builder.add_repository(repository, source_info) with patch('buildtool.bom_commands.now') as mock_now: mock_now.return_value = datetime.datetime(2018, 1, 2, 3, 4, 5) bom = builder.build() golden_bom['version'] = 'patch-OptionBuildNumber' golden_bom['timestamp'] = '2018-01-02 03:04:05' golden_bom['services'][NORMAL_SERVICE]['version'] = ( PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber') golden_bom['services'][OUTLIER_SERVICE]['version'] = ( PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber') golden_bom['services']['monitoring-third-party']['version'] = ( PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber') golden_bom['artifactSources'] = { 'debianRepository': 'https://dl.bintray.com/%s/%s' % (options.bintray_org, options.bintray_debian_repository), 'dockerRegistry': options.docker_registry, 'googleImageProject': options.publish_gce_image_project, 'gitPrefix': os.path.dirname(self.repo_commit_map[NORMAL_REPO]['ORIGIN']) } for key, value in bom['services'].items(): print '**** {0} -> {1}'.format(key, value) print ' ==>> {0}'.format(golden_bom['services'][key]) self.assertEquals(value, golden_bom['services'][key]) for key, value in bom.items(): self.assertEquals(value, golden_bom[key]) self.assertEquals(golden_bom, bom)
def _ensure_templates_directory(self): """Initialize the templates_directory, pulling the repo if needed.""" scm = BranchSourceCodeManager( make_options_with_fallback(self.options), self.get_input_dir()) repository = scm.make_repository_spec(SPINNAKER_GITHUB_IO_REPOSITORY_NAME) # These documents arent tied to version control, especially since they are # published to a different repository. scm.ensure_git_path(repository) self.__templates_directory = os.path.join( repository.git_dir, '_api_templates')
def test_rebuild(self): test_root = self.test_root options = self.options options.git_branch = 'master' options.github_owner = 'default' options.github_disable_upstream_push = True options.build_number = 'UpdatedBuildNumber' scm = BranchSourceCodeManager(options, test_root) builder = BomBuilder.new_from_bom(options, scm, MetricsManager.singleton(), self.golden_bom) repository = scm.make_repository_spec(OUTLIER_REPO) scm.ensure_git_path(repository) scm.git.check_run(repository.git_dir, 'checkout ' + PATCH_BRANCH) summary = scm.git.collect_repository_summary(repository.git_dir) source_info = SourceInfo('SourceInfoBuildNumber', summary) builder.add_repository(repository, source_info) with patch('buildtool.bom_commands.now') as mock_now: mock_now.return_value = datetime.datetime(2018, 1, 2, 3, 4, 5) bom = builder.build() updated_service = bom['services'][OUTLIER_SERVICE] self.assertEqual( updated_service, { 'commit': self.repo_commit_map[OUTLIER_REPO][PATCH_BRANCH], 'version': PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber' }) # The bom should be the same as before, but with new timestamp/version # and our service updated. And the artifactSources to our configs. updated_bom = dict(self.golden_bom) updated_bom['timestamp'] = '2018-01-02 03:04:05' updated_bom['version'] = 'master-UpdatedBuildNumber' updated_bom['services'][OUTLIER_SERVICE] = updated_service updated_bom['artifactSources'] = { 'debianRepository': 'https://dl.bintray.com/%s/%s' % (options.bintray_org, options.bintray_debian_repository), 'dockerRegistry': options.docker_registry, 'googleImageProject': options.publish_gce_image_project, 'gitPrefix': self.golden_bom['artifactSources']['gitPrefix'] } for key, value in updated_bom.items(): self.assertEqual(value, bom[key]) self.assertEqual(updated_bom, bom)
def __init__(self, factory, options, **kwargs): super(PublishSpinnakerCommand, self).__init__(factory, options, **kwargs) check_options_set( options, [ "spinnaker_version", "spinnaker_release_alias", "bom_version", "changelog_gist_url", "github_owner", "min_halyard_version", ], ) major, minor, _ = self.options.spinnaker_version.split(".") self.__branch = "release-{major}.{minor}.x".format(major=major, minor=minor) options_copy = copy.copy(options) self.__bom_scm = BomSourceCodeManager(options_copy, self.get_input_dir()) self.__hal = HalRunner(options) self.__git = GitRunner(options) self.__hal.check_property("spinnaker.config.input.bucket", options.halyard_bom_bucket) if options.only_repositories: self.__only_repositories = options.only_repositories.split(",") else: self.__only_repositories = [] options_copy.git_branch = self.__branch self.__branch_scm = BranchSourceCodeManager(options_copy, self.get_input_dir())
def __init__(self, factory, options, **kwargs): options_copy = copy.copy(options) options_copy.bom_path = None options_copy.bom_version = None options_copy.git_branch = 'master' options_copy.github_hostname = 'github.com' # Overrides later if --git_allow_publish_master_branch is false super(PublishHalyardCommand, self).__init__(factory, options_copy, **kwargs) check_options_set(options, ['halyard_version']) match = re.match(r'(\d+)\.(\d+)\.(\d+)-\d+', options.halyard_version) if match is None: raise_and_log_error( ConfigError('--halyard_version={version} is not X.Y.Z-<buildnum>' .format(version=options.halyard_version))) self.__stable_version = '{major}.{minor}.{patch}'.format( major=match.group(1), minor=match.group(2), patch=match.group(3)) self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir()) self.__hal = HalRunner(options_copy) self.__gradle = GradleRunner(options_copy, self.__scm, self.metrics) self.__halyard_repo_md_path = os.path.join('docs', 'commands.md') dash = self.options.halyard_version.find('-') semver_str = self.options.halyard_version[0:dash] semver_parts = semver_str.split('.') if len(semver_parts) != 3: raise_and_log_error( ConfigError('Expected --halyard_version in the form X.Y.Z-N')) self.__release_branch = 'release-{maj}.{min}.x'.format( maj=semver_parts[0], min=semver_parts[1]) self.__release_tag = 'version-' + semver_str self.__release_version = semver_str
def test_pull_repository_fallback_branch(self): test_root = os.path.join(self.base_temp_dir, "fallback_test") unique_branch = "RepoTwo-branch" options = make_default_options() options.git_branch = unique_branch options.git_fallback_branch = "master" options.build_number = "pull_repository_fallback_buildnumber" scm = BranchSourceCodeManager(options, test_root) for repo_name, origin in self.ORIGIN_URLS.items(): repository = scm.make_repository_spec(repo_name, origin=origin) scm.ensure_local_repository(repository) git_dir = repository.git_dir want_branch = unique_branch if repository.name == "RepoTwo" else "master" in_branch = scm.git.query_local_repository_branch(git_dir) self.assertEqual(want_branch, in_branch)
def init_argparser(self, parser, defaults): """Implements CommandFactory interface.""" # For the spinnaker.github.io repository BranchSourceCodeManager.add_parser_args(parser, defaults) self.add_argument( parser, 'max_wait_secs_startup', defaults, 210, type=int, help='Number of seconds to wait for gate server to startup' ' before giving up and timing out.') self.add_argument( parser, 'swagger_codegen_cli_jar_path', defaults, None, help='The location of the swagger-codegen-cli jarfile.' ' The file can be obtained from' ' http://central.maven.org/maven2/io/swagger/swagger-codegen-cli' '/2.2.3/swagger-codegen-cli-2.2.3.jar') super(BuildApiDocsFactory, self).init_argparser(parser, defaults)
def test_build(self): test_root = self.test_root options = self.options options.git_branch = PATCH_BRANCH options.github_owner = 'default' options.github_disable_upstream_push = True scm = BranchSourceCodeManager(options, test_root) golden_bom = dict(self.golden_bom) builder = BomBuilder.new_from_bom( options, scm, MetricsManager.singleton(), golden_bom) source_repositories = [scm.make_repository_spec(name) for name in ALL_STANDARD_TEST_BOM_REPO_NAMES] for repository in source_repositories: scm.ensure_git_path(repository) summary = scm.git.collect_repository_summary(repository.git_dir) source_info = SourceInfo('SourceInfoBuildNumber', summary) builder.add_repository(repository, source_info) with patch('buildtool.bom_commands.now') as mock_now: mock_now.return_value = datetime.datetime(2018, 1, 2, 3, 4, 5) bom = builder.build() golden_bom['version'] = 'patch-OptionBuildNumber' golden_bom['timestamp'] = '2018-01-02 03:04:05' golden_bom['services'][NORMAL_SERVICE]['version'] = ( PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber') golden_bom['services'][OUTLIER_SERVICE]['version'] = ( PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber') golden_bom['services']['monitoring-third-party']['version'] = ( PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber') golden_bom['artifactSources'] = { 'debianRepository': 'https://dl.bintray.com/%s/%s' % ( options.bintray_org, options.bintray_debian_repository), 'dockerRegistry': options.docker_registry, 'googleImageProject': options.publish_gce_image_project, 'gitPrefix': os.path.dirname(self.repo_commit_map[NORMAL_REPO]['ORIGIN']) } for key, value in bom['services'].items(): self.assertEqual(value, golden_bom['services'][key]) for key, value in bom.items(): self.assertEqual(value, golden_bom[key]) self.assertEqual(golden_bom, bom)
def test_rebuild(self): test_root = self.test_root options = self.options options.git_branch = 'master' options.github_owner = 'default' options.github_disable_upstream_push = True options.build_number = 'UpdatedBuildNumber' scm = BranchSourceCodeManager(options, test_root) builder = BomBuilder.new_from_bom( options, scm, MetricsManager.singleton(), self.golden_bom) repository = scm.make_repository_spec(OUTLIER_REPO) scm.ensure_git_path(repository) scm.git.check_run(repository.git_dir, 'checkout ' + PATCH_BRANCH) summary = scm.git.collect_repository_summary(repository.git_dir) source_info = SourceInfo('SourceInfoBuildNumber', summary) builder.add_repository(repository, source_info) with patch('buildtool.bom_commands.now') as mock_now: mock_now.return_value = datetime.datetime(2018, 1, 2, 3, 4, 5) bom = builder.build() updated_service = bom['services'][OUTLIER_SERVICE] self.assertEqual(updated_service, { 'commit': self.repo_commit_map[OUTLIER_REPO][PATCH_BRANCH], 'version': PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber' }) # The bom should be the same as before, but with new timestamp/version # and our service updated. And the artifactSources to our configs. updated_bom = dict(self.golden_bom) updated_bom['timestamp'] = '2018-01-02 03:04:05' updated_bom['version'] = 'master-UpdatedBuildNumber' updated_bom['services'][OUTLIER_SERVICE] = updated_service updated_bom['artifactSources'] = { 'debianRepository': 'https://dl.bintray.com/%s/%s' % ( options.bintray_org, options.bintray_debian_repository), 'dockerRegistry': options.docker_registry, 'googleImageProject': options.publish_gce_image_project, 'gitPrefix': self.golden_bom['artifactSources']['gitPrefix'] } for key, value in updated_bom.items(): self.assertEqual(value, bom[key]) self.assertEqual(updated_bom, bom)
def __init__(self, factory, options, **kwargs): options_copy = copy.copy(options) options_copy.git_branch = 'master' options_copy.github_hostname = 'github.com' super(PublishSpinCommand, self).__init__(factory, options_copy, **kwargs) check_options_set(options, ['spin_version']) # Ensure we have a version to promote. bom_contents = BomSourceCodeManager.load_bom(options_copy) gate_entry = bom_contents.get('services', {}).get('gate', {}) if not gate_entry: raise_and_log_error( ConfigError('No gate service entry found in bom {}'.format(bom_contents))) self.__spinnaker_version = options.bom_version or bom_contents['version'] self.__gate_version = gate_entry['version'] self.__stable_version = None # Set after promote_spin. self.__no_changes = False # Set after promote_spin. self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir()) self.__gcs_uploader = SpinGcsUploader(options)
def __init__(self, factory, options, **kwargs): super(PublishApiDocsCommand, self).__init__( factory, options, **kwargs) docs_dir = self.get_output_dir(command=BUILD_DOCS_COMMAND) self.__html_path = os.path.join(os.path.abspath(docs_dir), 'index.html') self._check_args() self.__scm = BranchSourceCodeManager( make_options_with_fallback(self.options), self.get_input_dir()) self.__docs_repository = self.__scm.make_repository_spec( SPINNAKER_GITHUB_IO_REPOSITORY_NAME)
def test_check_repository_branch(self): self.options.git_branch = UNTAGGED_BRANCH test_root = os.path.join(self.base_temp_dir, 'check_branch_test') scm = BranchSourceCodeManager(self.options, test_root) repository_name = 'RepoOne' origin = self.ORIGIN_URLS[repository_name] repository = scm.make_repository_spec(repository_name, origin=origin, upstream=None) scm.ensure_local_repository(repository) scm.check_repository_is_current(repository) scm.git.check_run(repository.git_dir, 'checkout master') self.assertRaises(UnexpectedError, scm.check_repository_is_current, repository)
def __init__(self, factory, options, **kwargs): super(BuildSpinCommand, self).__init__(factory, options, source_repository_names=SPIN_REPOSITORY_NAMES, **kwargs) options_copy = copy.copy(options) self.__gcs_uploader = SpinGcsUploader(options) self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir()) self.__build_version = None # recorded after build bom_contents = BomSourceCodeManager.load_bom(options) gate_entry = bom_contents.get('services', {}).get('gate', {}) if not gate_entry: raise_and_log_error( ConfigError('No gate service entry found in bom {}'.format( bom_contents))) self.__gate_version = gate_entry['version']
def __init__(self, factory, options, **kwargs): options_copy = copy.copy(options) options_copy.git_branch = 'master' options_copy.github_hostname = 'github.com' super(PublishSpinCommand, self).__init__(factory, options_copy, **kwargs) check_options_set(options, ['spin_version']) # Ensure we have a version to promote. bom_contents = BomSourceCodeManager.load_bom(options_copy) gate_entry = bom_contents.get('services', {}).get('gate', {}) if not gate_entry: raise_and_log_error( ConfigError('No gate service entry found in bom {}'.format(bom_contents))) self.__spinnaker_version = options.bom_version or bom_contents['version'] self.__gate_version = gate_entry['version'] self.__stable_version = None # Set after promote_spin. self.__no_changes = False # Set after promote_spin. self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir()) self.__gcs_uploader = SpinGcsUploader(options)
def test_build(self): test_root = self.test_root options = self.options options.git_branch = PATCH_BRANCH options.github_owner = "default" options.github_disable_upstream_push = True scm = BranchSourceCodeManager(options, test_root) golden_bom = dict(self.golden_bom) builder = BomBuilder.new_from_bom(options, scm, MetricsManager.singleton(), golden_bom) source_repositories = [ scm.make_repository_spec(name) for name in ALL_STANDARD_TEST_BOM_REPO_NAMES ] for repository in source_repositories: scm.ensure_git_path(repository) summary = scm.git.collect_repository_summary(repository.git_dir) source_info = SourceInfo("SourceInfoBuildNumber", summary) builder.add_repository(repository, source_info) with patch("buildtool.bom_commands.now") as mock_now: mock_now.return_value = datetime.datetime(2018, 1, 2, 3, 4, 5) bom = builder.build() golden_bom["version"] = "OptionBuildNumber" golden_bom["timestamp"] = "2018-01-02 03:04:05" golden_bom["services"][NORMAL_SERVICE]["version"] = BASE_VERSION_NUMBER golden_bom["services"][OUTLIER_SERVICE][ "version"] = BASE_VERSION_NUMBER golden_bom["services"]["monitoring-third-party"][ "version"] = BASE_VERSION_NUMBER golden_bom["artifactSources"] = { "gitPrefix": os.path.dirname(self.repo_commit_map[NORMAL_REPO]["ORIGIN"]), "debianRepository": SPINNAKER_DEBIAN_REPOSITORY, "dockerRegistry": SPINNAKER_DOCKER_REGISTRY, "googleImageProject": SPINNAKER_GOOGLE_IMAGE_PROJECT, } for key, value in bom["services"].items(): # gate has extra commit on branch so commit id's should not match if key in ["gate", "monitoring-daemon", "monitoring-third-party"]: self.assertNotEqual( value, golden_bom["services"][key], msg="key: {} - value: {}".format(key, value), ) else: self.assertEqual( value, golden_bom["services"][key], msg="key: {} - value: {}".format(key, value), ) for key, value in bom.items(): if key != "services": self.assertEqual(value, golden_bom[key])
class PublishSpinCommand(CommandProcessor): """Publish Spin CLI version to the public repository.""" def __init__(self, factory, options, **kwargs): options_copy = copy.copy(options) options_copy.git_branch = 'master' options_copy.github_hostname = 'github.com' super(PublishSpinCommand, self).__init__(factory, options_copy, **kwargs) check_options_set( options, ['spin_version']) # Ensure we have a version to promote. bom_contents = BomSourceCodeManager.load_bom(options_copy) gate_entry = bom_contents.get('services', {}).get('gate', {}) if not gate_entry: raise_and_log_error( ConfigError('No gate service entry found in bom {}'.format( bom_contents))) self.__spinnaker_version = options.bom_version or bom_contents[ 'version'] self.__gate_version = gate_entry['version'] self.__stable_version = None # Set after promote_spin. self.__no_changes = False # Set after promote_spin. self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir()) self.__gcs_uploader = SpinGcsUploader(options) def push_tag_and_branch(self, repository): """Pushes a stable branch and git version tag to the origin repository.""" if self.__no_changes: logging.info( 'No changes in spin since last tag, skipping branch and tag push.' ) return git_dir = repository.git_dir git = self.__scm.git match = re.match(r'(\d+)\.(\d+)\.(\d+)-\d+', self.__gate_version) if match is None: raise_and_log_error( ConfigError( 'gate version {version} is not X.Y.Z-<buildnum>'.format( version=self.__gate_version))) semver_parts = self.__spinnaker_version.split('.') if len(semver_parts) != 3: raise_and_log_error( ConfigError( 'Expected spinnaker version in the form X.Y.Z-N, got {}'. format(self.__spinnaker_version))) release_branch = 'release-{maj}.{min}.x'.format(maj=semver_parts[0], min=semver_parts[1]) release_tag = 'version-' + self.__stable_version logging.info('Pushing branch=%s and tag=%s to %s', release_branch, release_tag, repository.origin) git.check_run_sequence(git_dir, [ 'checkout -b ' + release_branch, 'push origin ' + release_branch, 'tag ' + release_tag, 'push origin ' + release_tag ]) def promote_spin(self, repository): """Promote an existing build to become the spin CLI stable version.""" git_dir = repository.git_dir git = self.__scm.git match = re.match(r'(\d+)\.(\d+)\.(\d+)-\d+', self.__gate_version) gate_major = match.group(1) gate_min = match.group(2) if match is None: raise_and_log_error( ConfigError( 'gate version {version} is not X.Y.Z-<buildnum>'.format( version=self.__gate_version))) semver_parts = self.__spinnaker_version.split('.') if len(semver_parts) != 3: raise_and_log_error( ConfigError('Expected spinnaker version in the form X.Y.Z-N')) # Note: spin CLI is coupled to the Gate major and minor version. # Gate is a routing server, so features and breaking changes in Gate # must be reflected in spin since it is a client. We pin only the major # and minor versions so fixes (thus patch version) are decoupled between # the two. patch = '0' # Patch is reset on a new Gate major or minor. tag_matcher = re.compile(r'version-{maj}.{min}.(\d+)'.format( maj=gate_major, min=gate_min)) tags = git.fetch_tags(git_dir) tag_matches = [ tag_matcher.match(t) for t in tags if tag_matcher.match(t) ] if tag_matches: patch_versions = [int(m.group(1)) for m in tag_matches] max_patch = max(patch_versions) last_tag = 'version-{maj}.{min}.{max_patch}'.format( maj=gate_major, min=gate_min, max_patch=max_patch) self.__no_changes = git.query_local_repository_commit_id( git_dir) == git.query_commit_at_tag(git_dir, last_tag) patch = str(max_patch + 1) self.__stable_version = '{major}.{minor}.{patch}'.format( major=match.group(1), minor=match.group(2), patch=patch) candidate = self.options.spin_version if self.__no_changes: logging.info( 'No changes in spin since last tag, skipping publish.') else: for d in DIST_ARCH_LIST: source = 'spin/{}/{}/{}/{}'.format(candidate, d.dist, d.arch, d.filename) dest = 'spin/{}/{}/{}/{}'.format(self.__stable_version, d.dist, d.arch, d.filename) self.__gcs_uploader.copy_file(source, dest) self.__update_release_latest_file(gate_major, gate_min) self.__update_global_latest_file() def __update_release_latest_file(self, gate_major, gate_minor): output_dir = self.get_output_dir() release_latest_file = '{}.{}.x-latest'.format(gate_major, gate_minor) with open(os.path.join(output_dir, release_latest_file), 'w') as lf: lf.write(self.__stable_version) release_latest_path = 'spin/{}.{}.x-latest'.format( gate_major, gate_minor) self.__gcs_uploader.upload_from_filename( release_latest_path, os.path.join(output_dir, release_latest_file)) def __update_global_latest_file(self): output_dir = self.get_output_dir() global_latest_version = self.__gcs_uploader.read_file('spin/latest') if LooseVersion( self.__stable_version) > LooseVersion(global_latest_version): with open(os.path.join(output_dir, 'latest'), 'w') as lf: lf.write(self.__stable_version) self.__gcs_uploader.upload_from_filename( 'spin/latest', os.path.join(output_dir, 'latest')) def _do_command(self): """Implements CommandProcessor interface.""" # TODO(jacobkiefer): Add spin CLI docs generation. repository = self.__scm.make_repository_spec('spin') git = self.__scm.git git.clone_repository_to_path(repository) self.promote_spin(repository) self.push_tag_and_branch(repository)
class PublishSpinCommand(CommandProcessor): """Publish Spin CLI version to the public repository.""" def __init__(self, factory, options, **kwargs): options_copy = copy.copy(options) options_copy.git_branch = 'master' options_copy.github_hostname = 'github.com' super(PublishSpinCommand, self).__init__(factory, options_copy, **kwargs) check_options_set(options, ['spin_version']) # Ensure we have a version to promote. bom_contents = BomSourceCodeManager.load_bom(options_copy) gate_entry = bom_contents.get('services', {}).get('gate', {}) if not gate_entry: raise_and_log_error( ConfigError('No gate service entry found in bom {}'.format(bom_contents))) self.__spinnaker_version = options.bom_version or bom_contents['version'] self.__gate_version = gate_entry['version'] self.__stable_version = None # Set after promote_spin. self.__no_changes = False # Set after promote_spin. self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir()) self.__gcs_uploader = SpinGcsUploader(options) def push_tag_and_branch(self, repository): """Pushes a stable branch and git version tag to the origin repository.""" if self.__no_changes: logging.info('No changes in spin since last tag, skipping branch and tag push.') return git_dir = repository.git_dir git = self.__scm.git match = re.match(r'(\d+)\.(\d+)\.(\d+)-\d+', self.__gate_version) if match is None: raise_and_log_error( ConfigError('gate version {version} is not X.Y.Z-<buildnum>' .format(version=self.__gate_version))) semver_parts = self.__spinnaker_version.split('.') if len(semver_parts) != 3: raise_and_log_error( ConfigError('Expected spinnaker version in the form X.Y.Z-N, got {}' .format(self.__spinnaker_version))) release_branch = 'origin/release-{maj}.{min}.x'.format( maj=semver_parts[0], min=semver_parts[1]) release_tag = 'version-' + self.__stable_version logging.info('Pushing branch=%s and tag=%s to %s', release_branch, release_tag, repository.origin) git.check_run_sequence( git_dir, [ 'checkout ' + release_branch, 'tag ' + release_tag, ]) git.push_tag_to_origin(git_dir, release_tag) def promote_spin(self, repository): """Promote an existing build to become the spin CLI stable version.""" git_dir = repository.git_dir git = self.__scm.git if len(self.__spinnaker_version.split('.')) != 3: raise_and_log_error( ConfigError('Expected spinnaker version in the form X.Y.Z-N')) next_spin_semver = bump_spin_patch(git, git_dir, self.__gate_version) gate_major = next_spin_semver.major gate_min = next_spin_semver.minor last_patch = str(max(next_spin_semver.patch - 1, 0)) self.__stable_version = next_spin_semver.to_version() logging.info('calculated new stable version: {}' .format(self.__stable_version)) # NOTE: major and minor for spin binaries are dependent on gate versions. last_tag = 'version-{maj}.{min}.{patch}'.format(maj=gate_major, min=gate_min, patch=last_patch) self.__no_changes = git.query_local_repository_commit_id(git_dir) == git.query_commit_at_tag(git_dir, last_tag) candidate = self.options.spin_version if self.__no_changes: logging.info('No changes in spin since last tag, skipping publish.') else: for d in DIST_ARCH_LIST: source = 'spin/{}/{}/{}/{}'.format(candidate, d.dist, d.arch, d.filename) dest = 'spin/{}/{}/{}/{}'.format(self.__stable_version, d.dist, d.arch, d.filename) self.__gcs_uploader.copy_file(source, dest) self.__update_release_latest_file(gate_major, gate_min) self.__update_global_latest_file() def __update_release_latest_file(self, gate_major, gate_minor): output_dir = self.get_output_dir() release_latest_file = '{}.{}.x-latest'.format(gate_major, gate_minor) with open(os.path.join(output_dir, release_latest_file), 'w') as lf: lf.write(self.__stable_version) release_latest_path = 'spin/{}.{}.x-latest'.format(gate_major, gate_minor) self.__gcs_uploader.upload_from_filename(release_latest_path, os.path.join(output_dir, release_latest_file)) def __update_global_latest_file(self): output_dir = self.get_output_dir() global_latest_version = self.__gcs_uploader.read_file('spin/latest') if LooseVersion(self.__stable_version) > LooseVersion(global_latest_version): with open(os.path.join(output_dir, 'latest'), 'w') as lf: lf.write(self.__stable_version) self.__gcs_uploader.upload_from_filename('spin/latest', os.path.join(output_dir, 'latest')) def _do_command(self): """Implements CommandProcessor interface.""" # TODO(jacobkiefer): Add spin CLI docs generation. repository = self.__scm.make_repository_spec('spin') git = self.__scm.git git.clone_repository_to_path(repository) self.promote_spin(repository) self.push_tag_and_branch(repository)
class PublishHalyardCommand(CommandProcessor): """Publish halyard version to the public repository.""" def __init__(self, factory, options, **kwargs): options_copy = copy.copy(options) options_copy.bom_path = None options_copy.bom_version = None options_copy.git_branch = 'master' options_copy.github_hostname = 'github.com' # Overrides later if --git_allow_publish_master_branch is false super(PublishHalyardCommand, self).__init__(factory, options_copy, **kwargs) check_options_set(options, ['halyard_version']) match = re.match(r'(\d+)\.(\d+)\.(\d+)-\d+', options.halyard_version) if match is None: raise_and_log_error( ConfigError('--halyard_version={version} is not X.Y.Z-<buildnum>' .format(version=options.halyard_version))) self.__stable_version = '{major}.{minor}.{patch}'.format( major=match.group(1), minor=match.group(2), patch=match.group(3)) self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir()) self.__hal = HalRunner(options_copy) self.__gradle = GradleRunner(options_copy, self.__scm, self.metrics) self.__halyard_repo_md_path = os.path.join('docs', 'commands.md') dash = self.options.halyard_version.find('-') semver_str = self.options.halyard_version[0:dash] semver_parts = semver_str.split('.') if len(semver_parts) != 3: raise_and_log_error( ConfigError('Expected --halyard_version in the form X.Y.Z-N')) self.__release_branch = 'release-{maj}.{min}.x'.format( maj=semver_parts[0], min=semver_parts[1]) self.__release_tag = 'version-' + semver_str self.__release_version = semver_str def determine_halyard_commit(self): """Determine the commit_id that we want to publish.""" options = self.options versions_url = options.halyard_version_commits_url if not versions_url: versions_url = '{base}/{filename}'.format( base=options.halyard_bucket_base_url, filename=BuildHalyardCommand.HALYARD_VERSIONS_BASENAME) if os.path.exists(versions_url): logging.debug('Loading halyard version info from file %s', versions_url) with open(versions_url, 'r') as stream: version_data = stream.read() else: logging.debug('Loading halyard version info from bucket %s', versions_url) version_data = check_subprocess( 'gsutil cat {url}'.format(url=versions_url)) commit = yaml.safe_load(version_data).get(options.halyard_version) if commit is None: raise_and_log_error( ConfigError('Unknown halyard version "{version}" in "{url}"'.format( version=options.halyard_version, url=versions_url))) return commit def _prepare_repository(self): """Prepare a local repository to build for release. Were rebuilding it only to have nebula give a new distribution tag. However we will also use the repository to tag and branch the release into github so want to at least clone the repo regardless. """ logging.debug('Preparing repository for publishing a halyard release.') commit = self.determine_halyard_commit() repository = self.__scm.make_repository_spec( SPINNAKER_HALYARD_REPOSITORY_NAME, commit_id=commit) git_dir = repository.git_dir if os.path.exists(git_dir): logging.info('Deleting existing %s to build fresh.', git_dir) shutil.rmtree(git_dir) git = self.__scm.git git.clone_repository_to_path(repository, commit=commit) return repository def _promote_halyard(self, repository): """Promote an existing build to become the halyard stable version.""" options = self.options logfile = self.get_logfile_path('promote-all') env = dict(os.environ) env.update({ 'PUBLISH_HALYARD_BUCKET_BASE_URL': options.halyard_bucket_base_url, 'PUBLISH_HALYARD_DOCKER_IMAGE_BASE': options.halyard_docker_image_base }) check_subprocesses_to_logfile( 'Promote Halyard', logfile, ['gcloud docker -a', # if repo is private it needs authenticated './release/promote-all.sh {candidate} {stable}'.format( candidate=options.halyard_version, stable=self.__stable_version), './release/promote-all.sh {candidate} stable'.format( candidate=options.halyard_version)], env=env, cwd=repository.git_dir) def _build_release(self, repository): """Rebuild the actual release debian package. We dont necessarily need to rebuild here. We just need to push as debian to the "-stable". However there isnt an easy way to do this. Note that this is not the promoted version. For safety[*] and simplicity we'll promote the candidate whose version was used to build this. Ideally this function can go away. [*] Safety because the candidate was tested whereas this build was not. """ # Ideally we would just modify the existing bintray version to add # *-stable to the distributions, however it does not appear possible # to patch the debian attributes of a bintray version, only the # version metadata. Therefore, we'll rebuild it. # Alternatively we could download the existing and push a new one, # however I dont see how to get at the existing debian metadata and # dont want to ommit something git_dir = repository.git_dir summary = self.__scm.git.collect_repository_summary(git_dir) args = self.__gradle.get_common_args() args.extend(self.__gradle.get_debian_args( 'trusty-stable,xenial-stable,bionic-stable')) build_number = self.options.build_number self.__gradle.check_run( args, self, repository, 'candidate', 'build-release', version=self.__release_version, build_number=build_number, gradle_dir=git_dir) info_path = os.path.join(self.get_output_dir(), 'halyard_info.yml') logging.debug('Writing build information to %s', info_path) write_to_path(summary.to_yaml(), info_path) def write_target_docs(self, source_repository, target_repository): source_path = os.path.join(source_repository.git_dir, self.__halyard_repo_md_path) target_rel_path = os.path.join('reference', 'halyard', 'commands.md') target_path = os.path.join(target_repository.git_dir, target_rel_path) now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') logging.debug('Writing documentation into %s', target_path) header = textwrap.dedent( """\ --- layout: single title: "Commands" sidebar: nav: reference --- Published: {now} """.format(now=now)) with open(source_path, 'r') as source: body = source.read() with open(target_path, 'w') as stream: stream.write(header) stream.write(body) return target_rel_path def push_docs(self, repository): base_branch = 'master' target_repository = self.__scm.make_repository_spec( SPINNAKER_GITHUB_IO_REPOSITORY_NAME) self.__scm.ensure_git_path(target_repository) target_rel_path = self.write_target_docs(repository, target_repository) if self.options.git_allow_publish_master_branch: head_branch = 'master' branch_flag = '' else: head_branch = self.__release_version + '-haldocs' branch_flag = '-b' logging.debug('Commiting changes into local repository "%s" branch=%s', target_repository.git_dir, head_branch) git_dir = target_repository.git_dir message = 'docs(halyard): ' + self.__release_version local_git_commands = [ # These commands are accomodating to a branch already existing # because the branch is on the version, not build. A rejected # build for some reason that is re-tried will have the same version # so the branch may already exist from the earlier attempt. 'checkout ' + base_branch, 'checkout {flag} {branch}'.format( flag=branch_flag, branch=head_branch), 'add ' + target_rel_path, ] logging.debug('Commiting changes into local repository "%s" branch=%s', target_repository.git_dir, head_branch) git = self.__scm.git git.check_run_sequence(git_dir, local_git_commands) git.check_commit_or_no_changes( git_dir, '-m "{msg}" {path}'.format(msg=message, path=target_rel_path)) logging.info('Pushing halyard docs to %s branch="%s"', target_repository.origin, head_branch) git.push_branch_to_origin( target_repository.git_dir, branch=head_branch, force=True) def _do_command(self): """Implements CommandProcessor interface.""" repository = self._prepare_repository() self._build_release(repository) self._promote_halyard(repository) build_halyard_docs(self, repository) self.push_docs(repository) self.push_tag_and_branch(repository) self.__hal.publish_halyard_release(self.__release_version) def push_tag_and_branch(self, repository): """Pushes a stable branch and git version tag to the origin repository.""" git_dir = repository.git_dir git = self.__scm.git release_url = git.determine_push_url(repository) logging.info('Pushing branch=%s and tag=%s to %s', self.__release_branch, self.__release_tag, release_url) existing_commit = git.query_commit_at_tag(git_dir, self.__release_tag) if existing_commit: want_commit = git.query_local_repository_commit_id(git_dir) if want_commit == existing_commit: logging.debug('Already have "%s" at %s', self.__release_tag, want_commit) return git.check_run_sequence( git_dir, [ 'checkout -b ' + self.__release_branch, 'remote add release ' + release_url, 'push release ' + self.__release_branch, 'tag ' + self.__release_tag, 'push release ' + self.__release_tag ])
class PublishSpinCommand(CommandProcessor): """Publish Spin CLI version to the public repository.""" def __init__(self, factory, options, **kwargs): options_copy = copy.copy(options) options_copy.git_branch = 'master' options_copy.github_hostname = 'github.com' super(PublishSpinCommand, self).__init__(factory, options_copy, **kwargs) check_options_set( options, ['spin_version']) # Ensure we have a version to promote. bom_contents = BomSourceCodeManager.load_bom(options_copy) gate_entry = bom_contents.get('services', {}).get('gate', {}) if not gate_entry: raise_and_log_error( ConfigError('No gate service entry found in bom {}'.format( bom_contents))) self.__spinnaker_version = options.bom_version or bom_contents[ 'version'] self.__gate_version = gate_entry['version'] self.__stable_version = None # Set after promote_spin. self.__no_changes = False # Set after promote_spin. self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir()) self.__gcs_uploader = SpinGcsUploader(options) def push_tag_and_branch(self, repository): """Pushes a stable branch and git version tag to the origin repository.""" if self.__no_changes: logging.info( 'No changes in spin since last tag, skipping branch and tag push.' ) return git_dir = repository.git_dir git = self.__scm.git match = re.match(r'(\d+)\.(\d+)\.(\d+)-\d+', self.__gate_version) if match is None: raise_and_log_error( ConfigError( 'gate version {version} is not X.Y.Z-<buildnum>'.format( version=self.__gate_version))) semver_parts = self.__spinnaker_version.split('.') if len(semver_parts) != 3: raise_and_log_error( ConfigError( 'Expected spinnaker version in the form X.Y.Z-N, got {}'. format(self.__spinnaker_version))) release_branch = 'origin/release-{maj}.{min}.x'.format( maj=semver_parts[0], min=semver_parts[1]) release_tag = 'version-' + self.__stable_version logging.info('Pushing branch=%s and tag=%s to %s', release_branch, release_tag, repository.origin) git.check_run_sequence(git_dir, [ 'checkout ' + release_branch, 'tag ' + release_tag, ]) git.push_tag_to_origin(git_dir, release_tag) def promote_spin(self, repository): """Promote an existing build to become the spin CLI stable version.""" git_dir = repository.git_dir git = self.__scm.git if len(self.__spinnaker_version.split('.')) != 3: raise_and_log_error( ConfigError('Expected spinnaker version in the form X.Y.Z-N')) next_spin_semver = bump_spin_patch(git, git_dir, self.__gate_version) gate_major = next_spin_semver.major gate_min = next_spin_semver.minor last_patch = str(max(next_spin_semver.patch - 1, 0)) self.__stable_version = next_spin_semver.to_version() logging.info('calculated new stable version: {}'.format( self.__stable_version)) # NOTE: major and minor for spin binaries are dependent on gate versions. last_tag = 'version-{maj}.{min}.{patch}'.format(maj=gate_major, min=gate_min, patch=last_patch) self.__no_changes = git.query_local_repository_commit_id( git_dir) == git.query_commit_at_tag(git_dir, last_tag) candidate = self.options.spin_version if self.__no_changes: logging.info( 'No changes in spin since last tag, skipping publish.') else: for d in DIST_ARCH_LIST: source = 'spin/{}/{}/{}/{}'.format(candidate, d.dist, d.arch, d.filename) dest = 'spin/{}/{}/{}/{}'.format(self.__stable_version, d.dist, d.arch, d.filename) self.__gcs_uploader.copy_file(source, dest) self.__update_release_latest_file(gate_major, gate_min) self.__update_global_latest_file() def __update_release_latest_file(self, gate_major, gate_minor): output_dir = self.get_output_dir() release_latest_file = '{}.{}.x-latest'.format(gate_major, gate_minor) with open(os.path.join(output_dir, release_latest_file), 'w') as lf: lf.write(self.__stable_version) release_latest_path = 'spin/{}.{}.x-latest'.format( gate_major, gate_minor) self.__gcs_uploader.upload_from_filename( release_latest_path, os.path.join(output_dir, release_latest_file)) def __update_global_latest_file(self): output_dir = self.get_output_dir() global_latest_version = self.__gcs_uploader.read_file('spin/latest') if LooseVersion( self.__stable_version) > LooseVersion(global_latest_version): with open(os.path.join(output_dir, 'latest'), 'w') as lf: lf.write(self.__stable_version) self.__gcs_uploader.upload_from_filename( 'spin/latest', os.path.join(output_dir, 'latest')) def _do_command(self): """Implements CommandProcessor interface.""" # TODO(jacobkiefer): Add spin CLI docs generation. repository = self.__scm.make_repository_spec('spin') git = self.__scm.git git.clone_repository_to_path(repository) self.promote_spin(repository) self.push_tag_and_branch(repository)
class PublishHalyardCommand(CommandProcessor): """Publish halyard version to the public repository.""" def __init__(self, factory, options, **kwargs): options_copy = copy.copy(options) options_copy.bom_path = None options_copy.bom_version = None options_copy.git_branch = 'master' # Overrides later if --git_allow_publish_master_branch is false super(PublishHalyardCommand, self).__init__(factory, options_copy, **kwargs) self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir()) self.__hal = HalRunner(options_copy) self.__gradle = GradleRunner(options_copy, self.__scm, self.metrics) self.__halyard_repo_md_path = os.path.join('docs', 'commands.md') dash = self.options.halyard_version.find('-') semver_str = self.options.halyard_version[0:dash] semver_parts = semver_str.split('.') if len(semver_parts) != 3: raise_and_log_error( ConfigError('Expected --halyard_version in the form X.Y.Z-N')) self.__release_branch = 'release-{maj}.{min}.x'.format( maj=semver_parts[0], min=semver_parts[1]) self.__release_tag = 'version-' + semver_str self.__release_version = semver_str def determine_commit(self, repository): """Determine the commit_id that we want to publish.""" if repository.name != 'halyard': raise_and_log_error( ConfigError('Unexpected repository "%s"' % repository.name)) options = self.options versions_url = options.halyard_version_commits_url if not versions_url: versions_url = '{base}/{filename}'.format( base=options.halyard_bucket_base_url, filename=BuildHalyardCommand.HALYARD_VERSIONS_BASENAME) if os.path.exists(versions_url): logging.debug('Loading halyard version info from file %s', versions_url) with open(versions_url, 'r') as stream: version_data = stream.read() else: logging.debug('Loading halyard version info from bucket %s', versions_url) version_data = check_subprocess( 'gsutil cat {url}'.format(url=versions_url)) commit = yaml.load(version_data).get(options.halyard_version) if commit is None: raise_and_log_error( ConfigError( 'Unknown halyard version "{version}" in "{url}"'.format( version=options.halyard_version, url=versions_url))) def _prepare_repository(self): """Prepare a local repository to build for release. Were rebuilding it only to have nebula give a new distribution tag. However we will also use the repository to tag and branch the release into github so want to at least clone the repo regardless. """ logging.debug('Preparing repository for publishing a halyard release.') repository = self.__scm.make_repository_spec( SPINNAKER_HALYARD_REPOSITORY_NAME) commit = self.determine_commit(repository) git_dir = repository.git_dir if os.path.exists(git_dir): logging.info('Deleting existing %s to build fresh.', git_dir) shutil.rmtree(git_dir) git = self.__scm.git git.clone_repository_to_path(repository, commit=commit) self.__scm.refresh_source_info(repository, self.options.build_number) return repository def _build_release(self, repository): """Rebuild the actual release. We dont necessarily need to rebuild here. We just need to push as debian to the "-stable". """ # Ideally we would just modify the existing bintray version to add # trusty-stable to the distributions, however it does not appear possible # to patch the debian attributes of a bintray version, only the # version metadata. Therefore, we'll rebuild it. # Alternatively we could download the existing and push a new one, # however I dont see how to get at the existing debian metadata and # dont want to ommit something git_dir = repository.git_dir summary = self.__scm.git.collect_repository_summary(git_dir) args = self.__gradle.get_common_args() args.extend( self.__gradle.get_debian_args('trusty-stable,xenial-stable')) build_number = self.options.build_number if not self.__gradle.consider_debian_on_bintray( repository, build_number=build_number): self.__gradle.check_run(args, self, repository, 'candidate', 'build-release', version=self.__release_version, build_number=build_number, gradle_dir=git_dir) info_path = os.path.join(self.get_output_dir(), 'halyard_info.yml') logging.debug('Writing build information to %s', info_path) write_to_path(summary.to_yaml(), info_path) def write_target_docs(self, source_repository, target_repository): source_path = os.path.join(source_repository.git_dir, self.__halyard_repo_md_path) target_rel_path = os.path.join('reference', 'halyard', 'commands.md') target_path = os.path.join(target_repository.git_dir, target_rel_path) now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') logging.debug('Writing documentation into %s', target_path) header = textwrap.dedent("""\ --- layout: single title: "Commands" sidebar: nav: reference --- Published: {now} """.format(now=now)) with open(source_path, 'r') as source: body = source.read() with open(target_path, 'w') as stream: stream.write(header) stream.write(body) return target_rel_path def push_docs(self, repository): base_branch = 'master' target_repository = self.__scm.make_repository_spec( SPINNAKER_GITHUB_IO_REPOSITORY_NAME) self.__scm.ensure_git_path(target_repository) target_rel_path = self.write_target_docs(repository, target_repository) if self.options.git_allow_publish_master_branch: head_branch = 'master' branch_flag = '' else: head_branch = self.__release_version + '-haldocs' branch_flag = '-b' logging.debug('Commiting changes into local repository "%s" branch=%s', target_repository.git_dir, head_branch) git_dir = target_repository.git_dir message = 'docs(halyard): ' + self.__release_version local_git_commands = [ # These commands are accomodating to a branch already existing # because the branch is on the version, not build. A rejected # build for some reason that is re-tried will have the same version # so the branch may already exist from the earlier attempt. 'checkout ' + base_branch, 'checkout {flag} {branch}'.format(flag=branch_flag, branch=head_branch), 'add ' + target_rel_path, 'commit -m "{msg}" {path}'.format(msg=message, path=target_rel_path), ] logging.debug('Commiting changes into local repository "%s" branch=%s', target_repository.git_dir, head_branch) git = self.__scm.git git.check_run_sequence(git_dir, local_git_commands) logging.info('Pushing halyard docs to %s branch="%s"', target_repository.origin, head_branch) git.push_branch_to_origin(target_repository.git_dir, branch=head_branch) def _do_command(self): """Implements CommandProcessor interface.""" repository = self._prepare_repository() self._build_release(repository) build_halyard_docs(self, repository) self.push_docs(repository) self.push_tag_and_branch(repository) self.__hal.publish_halyard_release(self.__release_version) def push_tag_and_branch(self, repository): """Pushes a stable branch and git version tag to the origin repository.""" git_dir = repository.git_dir git = self.__scm.git release_url = repository.origin logging.info('Pushing branch=%s and tag=%s to %s', self.__release_tag, self.__release_branch, release_url) git.check_run_sequence(git_dir, [ 'checkout -b ' + self.__release_branch, 'remote add release ' + release_url, 'push release ' + self.__release_branch, 'tag ' + self.__release_tag, 'push release ' + self.__release_tag ])
class PublishSpinnakerCommand(CommandProcessor): """"Implements the publish_spinnaker command.""" # pylint: disable=too-few-public-methods def __init__(self, factory, options, **kwargs): super(PublishSpinnakerCommand, self).__init__(factory, options, **kwargs) check_options_set(options, [ 'spinnaker_version', 'spinnaker_release_alias', 'bom_version', 'changelog_gist_url', 'github_owner', 'min_halyard_version' ]) major, minor, _ = self.options.spinnaker_version.split('.') self.__branch = 'release-{major}.{minor}.x'.format( major=major, minor=minor) options_copy = copy.copy(options) self.__scm = BomSourceCodeManager(options_copy, self.get_input_dir()) self.__hal = HalRunner(options) self.__git = GitRunner(options) self.__hal.check_property( 'spinnaker.config.input.bucket', options.halyard_bom_bucket) if options.only_repositories: self.__only_repositories = options.only_repositories.split(',') else: self.__only_repositories = [] options_copy.git_branch = self.__branch self.__process_scm = BranchSourceCodeManager( options_copy, self.get_input_dir()) def push_branches_and_tags(self, bom): """Update the release branches and tags in each of the BOM repositires.""" logging.info('Tagging each of the BOM service repos') # Run in two passes so we dont push anything if we hit a problem # in the tagging pass. Since we are spread against multiple repositiories, # we cannot do this atomically. The two passes gives us more protection # from a partial push due to errors in a repo. names_to_push = set([]) for which in ['tag', 'push']: for name, spec in bom['services'].items(): if name in ['monitoring-third-party', 'defaultArtifact']: # Ignore this, it is redundant to monitoring-daemon continue if name == 'monitoring-daemon': name = 'spinnaker-monitoring' if self.__only_repositories and name not in self.__only_repositories: logging.debug('Skipping %s because of --only_repositories', name) continue if spec is None: logging.warning('HAVE bom.services.%s = None', name) continue repository = self.__scm.make_repository_spec(name) self.__scm.ensure_local_repository(repository) if which == 'tag': added = self.__branch_and_tag_repository( repository, self.__branch) if added: names_to_push.add(name) else: self.__push_branch_and_maybe_tag_repository( repository, self.__branch, name in names_to_push) additional_repositories = list(SPINNAKER_PROCESS_REPOSITORY_NAMES) additional_repositories.extend(SPINNAKER_SHARED_REPOSITORY_NAMES) for name in additional_repositories: if self.__only_repositories and name not in self.__only_repositories: logging.debug('Skipping %s because of --only_repositories', name) continue repository = self.__process_scm.make_repository_spec(name) self.__process_scm.ensure_local_repository(repository) if self.__branch_and_tag_repository(repository, self.__branch): self.__push_branch_and_maybe_tag_repository( repository, self.__branch, True) def __already_have_tag(self, repository, tag): """Determine if we already have the tag in the repository.""" git_dir = repository.git_dir existing_commit = self.__git.query_commit_at_tag(git_dir, tag) if not existing_commit: return False want_commit = self.__git.query_local_repository_commit_id(git_dir) if want_commit == existing_commit: logging.debug('Already have "%s" at %s', tag, want_commit) return True raise_and_log_error( ConfigError( '"{tag}" already exists in "{repo}" at commit {have}, not {want}' .format(tag=tag, repo=git_dir, have=existing_commit, want=want_commit))) def __branch_and_tag_repository(self, repository, branch): """Create a branch and/or verison tag in the repository, if needed.""" version = self.__scm.determine_repository_version(repository) tag = 'version-' + version if self.__already_have_tag(repository, tag): return False self.__git.check_run(repository.git_dir, 'tag ' + tag) return True def __push_branch_and_maybe_tag_repository(self, repository, branch, also_tag): """Push the branch and verison tag to the origin.""" tag = 'version-' + self.__scm.determine_repository_version(repository) self.__git.push_branch_to_origin(repository.git_dir, branch) if also_tag: self.__git.push_tag_to_origin(repository.git_dir, tag) else: logging.info('%s was already tagged with "%s" -- skip', repository.git_dir, tag) def _do_command(self): """Implements CommandProcessor interface.""" options = self.options spinnaker_version = options.spinnaker_version options_copy = copy.copy(options) options_copy.git_branch = 'master' # push to master in spinnaker.github.io publish_changelog_command = PublishChangelogFactory().make_command( options_copy) changelog_gist_url = options.changelog_gist_url # Make sure changelog exists already. # If it does not then fail. try: logging.debug('Verifying changelog ready at %s', changelog_gist_url) urlopen(changelog_gist_url) except HTTPError: logging.error(exception_to_message) raise_and_log_error( ConfigError( 'Changelog gist "{url}" must exist before publising a release.' .format(url=changelog_gist_url), cause='ChangelogMissing')) bom = self.__hal.retrieve_bom_version(self.options.bom_version) bom['version'] = spinnaker_version bom_path = os.path.join(self.get_output_dir(), spinnaker_version + '.yml') write_to_path(yaml.safe_dump(bom, default_flow_style=False), bom_path) self.__hal.publish_bom_path(bom_path) self.push_branches_and_tags(bom) self.__hal.publish_spinnaker_release( spinnaker_version, options.spinnaker_release_alias, changelog_gist_url, options.min_halyard_version) logging.info('Publishing changelog') publish_changelog_command()
class PublishApiDocsCommand(CommandProcessor): """Publish built docs back up to the origin repository. This uses git_runner to checkout the SPINNAKER_GITHUB_IO_REPOSITORY that we'll be publishing docs to. It assumes that the documentation has already been built so that it could have been reviewed. """ def __init__(self, factory, options, **kwargs): super(PublishApiDocsCommand, self).__init__( factory, options, **kwargs) docs_dir = self.get_output_dir(command=BUILD_DOCS_COMMAND) self.__html_path = os.path.join(os.path.abspath(docs_dir), 'index.html') self._check_args() self.__scm = BranchSourceCodeManager( make_options_with_fallback(self.options), self.get_input_dir()) self.__docs_repository = self.__scm.make_repository_spec( SPINNAKER_GITHUB_IO_REPOSITORY_NAME) def _check_args(self): check_path_exists(self.__html_path, why='output from running "build_apidocs"') check_options_set(self.options, ['spinnaker_version']) def _do_command(self): """Implements CommandProcessor interface.""" self.__scm.ensure_git_path(self.__docs_repository, branch='master') base_branch = 'master' version = self.options.spinnaker_version if self.options.git_allow_publish_master_branch: head_branch = 'master' branch_flag = '' else: head_branch = version + '-apidocs' branch_flag = '-b' files_added = self._prepare_local_repository_files() git_dir = self.__docs_repository.git_dir message = 'docs(api): API Documentation for Spinnaker ' + version local_git_commands = [ # These commands are accomodating to a branch already existing # because the branch is on the version, not build. A rejected # build for some reason that is re-tried will have the same version # so the branch may already exist from the earlier attempt. 'checkout ' + base_branch, 'checkout {flag} {branch}'.format( flag=branch_flag, branch=head_branch), 'add ' + ' '.join([os.path.abspath(path) for path in files_added]), ] logging.debug('Commiting changes into local repository "%s" branch=%s', self.__docs_repository.git_dir, head_branch) git = self.__scm.git git.check_run_sequence(git_dir, local_git_commands) git.check_commit_or_no_changes(git_dir, '-m "{msg}"'.format(msg=message)) logging.info('Pushing branch="%s" into "%s"', head_branch, self.__docs_repository.origin) git.push_branch_to_origin(git_dir, branch=head_branch, force=True) def _prepare_local_repository_files(self): """Implements CommandProcessor interface.""" # And putting them into the SPINNAKER_GITHUB_IO_REPOSITORY_NAME # # NOTE(ewiseblatt): 20171218 # This is the current scheme, however I think this should read # os.path.join(git_dir, spinnaker_minor_branch, docs_path_in_repo) # where "spinnaker_minor_branch" is the version with the <PATCH> # replaced with 'x' target_path = os.path.join( self.__docs_repository.git_dir, 'reference', 'api', 'docs.html') logging.debug('Copying %s to %s', self.__html_path, target_path) shutil.copy2(self.__html_path, target_path) return [target_path]
def setUp(self): super(TestBomBuilder, self).setUp() self.test_root = os.path.join(self.base_temp_dir, self._testMethodName) self.scm = BranchSourceCodeManager(self.options, self.test_root)
def test_check_repository_commit(self): self.options.git_branch = UNTAGGED_BRANCH test_root = os.path.join(self.base_temp_dir, 'check_commit_test') scm = BranchSourceCodeManager(self.options, test_root) repository_name = 'RepoOne' origin = self.ORIGIN_URLS[repository_name] repository = scm.make_repository_spec(repository_name, origin=origin, upstream=None) scm.ensure_local_repository(repository) scm.check_repository_is_current(repository) # Create another repo pinned to this particular commit. commit = scm.git.query_local_repository_commit_id(repository.git_dir) repository_at_commit = scm.make_repository_spec(repository_name, origin=origin, upstream=None, commit_id=commit) # We're at the commit, so this should validate. scm.check_repository_is_current(repository_at_commit) # Make a change to the repo so commit differs. with open(os.path.join(repository.git_dir, 'scrap'), 'w') as stream: stream.write('hello') scm.git.check_run(repository.git_dir, 'add scrap') scm.git.check_run(repository.git_dir, 'commit -a -m "added file"') # We're still in the branch, so the branch only repository is still ok, # but no longer at the same commit, so the commit-pinned repository is bad. scm.check_repository_is_current(repository) self.assertRaises(UnexpectedError, scm.check_repository_is_current, repository_at_commit)
def test_rebuild(self): test_root = self.test_root options = self.options options.git_branch = "master" options.github_owner = "default" options.github_disable_upstream_push = True options.build_number = "UpdatedBuildNumber" scm = BranchSourceCodeManager(options, test_root) builder = BomBuilder.new_from_bom(options, scm, MetricsManager.singleton(), self.golden_bom) repository = scm.make_repository_spec(OUTLIER_REPO) scm.ensure_git_path(repository) scm.git.check_run(repository.git_dir, "checkout " + PATCH_BRANCH) summary = scm.git.collect_repository_summary(repository.git_dir) source_info = SourceInfo("SourceInfoBuildNumber", summary) builder.add_repository(repository, source_info) with patch("buildtool.bom_commands.now") as mock_now: mock_now.return_value = datetime.datetime(2018, 1, 2, 3, 4, 5) bom = builder.build() updated_service = bom["services"][OUTLIER_SERVICE] # OUTLIER_REPO hasn't been tagged since extra commits so bom should be same self.assertNotEqual( updated_service, { "commit": self.repo_commit_map[OUTLIER_REPO][PATCH_BRANCH], "version": BASE_VERSION_NUMBER, }, ) # The bom should be the same as before, but with new timestamp/version # and our service unchanged. And the artifactSources to our configs. updated_bom = dict(self.golden_bom) updated_bom["timestamp"] = "2018-01-02 03:04:05" updated_bom["version"] = "UpdatedBuildNumber" updated_bom["artifactSources"] = { "gitPrefix": self.golden_bom["artifactSources"]["gitPrefix"], "debianRepository": SPINNAKER_DEBIAN_REPOSITORY, "dockerRegistry": SPINNAKER_DOCKER_REGISTRY, "googleImageProject": SPINNAKER_GOOGLE_IMAGE_PROJECT, } for key, value in bom["services"].items(): # monitoring-daemon has extra commit on branch so commit id's should not match if key in ["monitoring-daemon", "monitoring-third-party"]: self.assertNotEqual( value, updated_bom["services"][key], msg="key: {} - value: {}".format(key, value), ) else: self.assertEqual( value, updated_bom["services"][key], msg="key: {} - value: {}".format(key, value), ) for key, value in bom.items(): if key != "services": self.assertEqual(value, updated_bom[key])
class PublishHalyardCommand(CommandProcessor): """Publish halyard version to the public repository.""" def __init__(self, factory, options, **kwargs): options_copy = copy.copy(options) options_copy.bom_path = None options_copy.bom_version = None options_copy.git_branch = 'master' options_copy.github_hostname = 'github.com' # Overrides later if --git_allow_publish_master_branch is false super(PublishHalyardCommand, self).__init__(factory, options_copy, **kwargs) check_options_set(options, ['halyard_version']) match = re.match(r'(\d+)\.(\d+)\.(\d+)-\d+', options.halyard_version) if match is None: raise_and_log_error( ConfigError( '--halyard_version={version} is not X.Y.Z-<buildnum>'. format(version=options.halyard_version))) self.__stable_version = '{major}.{minor}.{patch}'.format( major=match.group(1), minor=match.group(2), patch=match.group(3)) self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir()) self.__hal = HalRunner(options_copy) self.__gradle = GradleRunner(options_copy, self.__scm, self.metrics) self.__halyard_repo_md_path = os.path.join('docs', 'commands.md') dash = self.options.halyard_version.find('-') semver_str = self.options.halyard_version[0:dash] semver_parts = semver_str.split('.') if len(semver_parts) != 3: raise_and_log_error( ConfigError('Expected --halyard_version in the form X.Y.Z-N')) self.__release_branch = 'release-{maj}.{min}.x'.format( maj=semver_parts[0], min=semver_parts[1]) self.__release_tag = 'version-' + semver_str self.__release_version = semver_str def determine_halyard_commit(self): """Determine the commit_id that we want to publish.""" options = self.options versions_url = options.halyard_version_commits_url if not versions_url: versions_url = '{base}/{filename}'.format( base=options.halyard_bucket_base_url, filename=BuildHalyardCommand.HALYARD_VERSIONS_BASENAME) if os.path.exists(versions_url): logging.debug('Loading halyard version info from file %s', versions_url) with open(versions_url, 'r') as stream: version_data = stream.read() else: logging.debug('Loading halyard version info from bucket %s', versions_url) gsutil_output = check_subprocess( 'gsutil cat {url}'.format(url=versions_url), stderr=subprocess.PIPE) # The latest version of gsutil prints a bunch of python warnings to stdout # (see b/152449160). This file is a series of lines that look like... # 0.41.0-180209172926: 05f1e832ab438e5a980d1102e84cdb348a0ab055 # ...so we'll just throw out any lines that don't start with digits. valid_lines = [ line for line in gsutil_output.splitlines() if line[0].isdigit() ] version_data = "\n".join(valid_lines) commit = yaml.safe_load(version_data).get(options.halyard_version) if commit is None: raise_and_log_error( ConfigError( 'Unknown halyard version "{version}" in "{url}"'.format( version=options.halyard_version, url=versions_url))) return commit def _prepare_repository(self): """Prepare a local repository to build for release. Were rebuilding it only to have nebula give a new distribution tag. However we will also use the repository to tag and branch the release into github so want to at least clone the repo regardless. """ logging.debug('Preparing repository for publishing a halyard release.') commit = self.determine_halyard_commit() repository = self.__scm.make_repository_spec( SPINNAKER_HALYARD_REPOSITORY_NAME, commit_id=commit) git_dir = repository.git_dir if os.path.exists(git_dir): logging.info('Deleting existing %s to build fresh.', git_dir) shutil.rmtree(git_dir) git = self.__scm.git git.clone_repository_to_path(repository, commit=commit) return repository def _promote_halyard(self, repository): """Promote an existing build to become the halyard stable version.""" options = self.options logfile = self.get_logfile_path('promote-all') env = dict(os.environ) env.update({ 'PUBLISH_HALYARD_ARTIFACT_DOCKER_IMAGE_SRC_BASE': options.halyard_artifact_registry_image_base, 'PUBLISH_HALYARD_BUCKET_BASE_URL': options.halyard_bucket_base_url, 'PUBLISH_HALYARD_DOCKER_IMAGE_BASE': options.halyard_docker_image_base }) check_subprocesses_to_logfile( 'Promote Halyard', logfile, [ 'gcloud docker -a', # if repo is private it needs authenticated './release/promote-all.sh {candidate} {stable}'.format( candidate=options.halyard_version, stable=self.__stable_version), './release/promote-all.sh {candidate} stable'.format( candidate=options.halyard_version) ], env=env, cwd=repository.git_dir) def _build_release(self, repository): """Rebuild the actual release debian package. We dont necessarily need to rebuild here. We just need to push as debian to the "-stable". However there isnt an easy way to do this. Note that this is not the promoted version. For safety[*] and simplicity we'll promote the candidate whose version was used to build this. Ideally this function can go away. [*] Safety because the candidate was tested whereas this build was not. """ # Ideally we would just modify the existing bintray version to add # *-stable to the distributions, however it does not appear possible # to patch the debian attributes of a bintray version, only the # version metadata. Therefore, we'll rebuild it. # Alternatively we could download the existing and push a new one, # however I dont see how to get at the existing debian metadata and # dont want to ommit something options = self.options git_dir = repository.git_dir summary = self.__scm.git.collect_repository_summary(git_dir) config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cloudbuild', 'debs.yml') substitutions = { '_BRANCH_NAME': options.git_branch, '_BRANCH_TAG': re.sub(r'\W', '_', options.git_branch), '_BUILD_NUMBER': options.build_number, '_IMAGE_NAME': 'halyard', '_VERSION': summary.version } # Convert it to the format expected by gcloud: "_FOO=bar,_BAZ=qux" substitutions_arg = ','.join('='.join((str(k), str(v))) for k, v in substitutions.items()) command = ('gcloud builds submit ' ' --account={account} --project={project}' ' --substitutions={substitutions_arg},' ' --config={config} .'.format( account=options.gcb_service_account, project=options.gcb_project, substitutions_arg=substitutions_arg, config=config_path)) logfile = self.get_logfile_path('build-deb') check_subprocesses_to_logfile('building deb with published version', logfile, [command], cwd=git_dir) def write_target_docs(self, source_repository, target_repository): source_path = os.path.join(source_repository.git_dir, self.__halyard_repo_md_path) target_rel_path = os.path.join('reference', 'halyard', 'commands.md') target_path = os.path.join(target_repository.git_dir, target_rel_path) now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') logging.debug('Writing documentation into %s', target_path) header = textwrap.dedent("""\ --- layout: single title: "Commands" sidebar: nav: reference --- Published: {now} """.format(now=now)) with open(source_path, 'r') as source: body = source.read() with open(target_path, 'w') as stream: stream.write(header) stream.write(body) return target_rel_path def push_docs(self, repository): base_branch = 'master' target_repository = self.__scm.make_repository_spec( SPINNAKER_GITHUB_IO_REPOSITORY_NAME) self.__scm.ensure_git_path(target_repository) target_rel_path = self.write_target_docs(repository, target_repository) if self.options.git_allow_publish_master_branch: head_branch = 'master' branch_flag = '' else: head_branch = self.__release_version + '-haldocs' branch_flag = '-b' logging.debug('Commiting changes into local repository "%s" branch=%s', target_repository.git_dir, head_branch) git_dir = target_repository.git_dir message = 'docs(halyard): ' + self.__release_version local_git_commands = [ # These commands are accomodating to a branch already existing # because the branch is on the version, not build. A rejected # build for some reason that is re-tried will have the same version # so the branch may already exist from the earlier attempt. 'checkout ' + base_branch, 'checkout {flag} {branch}'.format(flag=branch_flag, branch=head_branch), 'add ' + target_rel_path, ] logging.debug('Commiting changes into local repository "%s" branch=%s', target_repository.git_dir, head_branch) git = self.__scm.git git.check_run_sequence(git_dir, local_git_commands) git.check_commit_or_no_changes( git_dir, '-m "{msg}" {path}'.format(msg=message, path=target_rel_path)) logging.info('Pushing halyard docs to %s branch="%s"', target_repository.origin, head_branch) git.push_branch_to_origin(target_repository.git_dir, branch=head_branch, force=True) def _do_command(self): """Implements CommandProcessor interface.""" repository = self._prepare_repository() self._build_release(repository) self._promote_halyard(repository) build_halyard_docs(self, repository) self.push_docs(repository) self.push_tag_and_branch(repository) self.__hal.publish_halyard_release(self.__release_version) def push_tag_and_branch(self, repository): """Pushes a stable branch and git version tag to the origin repository.""" git_dir = repository.git_dir git = self.__scm.git release_url = git.determine_push_url(repository.origin) logging.info('Pushing branch=%s and tag=%s to %s', self.__release_branch, self.__release_tag, release_url) existing_commit = git.query_commit_at_tag(git_dir, self.__release_tag) if existing_commit: want_commit = git.query_local_repository_commit_id(git_dir) if want_commit == existing_commit: logging.debug('Already have "%s" at %s', self.__release_tag, want_commit) return git.check_run_sequence(git_dir, [ 'checkout -b ' + self.__release_branch, 'remote add release ' + release_url, 'push release ' + self.__release_branch, 'tag ' + self.__release_tag, 'push release ' + self.__release_tag ])
class PublishApiDocsCommand(CommandProcessor): """Publish built docs back up to the origin repository. This uses git_runner to checkout the SPINNAKER_GITHUB_IO_REPOSITORY that we'll be publishing docs to. It assumes that the documentation has already been built so that it could have been reviewed. """ def __init__(self, factory, options, **kwargs): super(PublishApiDocsCommand, self).__init__(factory, options, **kwargs) docs_dir = self.get_output_dir(command=BUILD_DOCS_COMMAND) self.__html_path = os.path.join(os.path.abspath(docs_dir), 'index.html') self._check_args() self.__scm = BranchSourceCodeManager( make_options_with_fallback(self.options), self.get_input_dir()) self.__docs_repository = self.__scm.make_repository_spec( SPINNAKER_GITHUB_IO_REPOSITORY_NAME) def _check_args(self): check_path_exists(self.__html_path, why='output from running "build_apidocs"') check_options_set(self.options, ['spinnaker_version']) def _do_command(self): """Implements CommandProcessor interface.""" self.__scm.ensure_git_path(self.__docs_repository, branch='master') base_branch = 'master' version = self.options.spinnaker_version if self.options.git_allow_publish_master_branch: head_branch = 'master' branch_flag = '' else: head_branch = version + '-apidocs' branch_flag = '-b' files_added = self._prepare_local_repository_files() git_dir = self.__docs_repository.git_dir message = 'docs(api): API Documentation for Spinnaker ' + version local_git_commands = [ # These commands are accomodating to a branch already existing # because the branch is on the version, not build. A rejected # build for some reason that is re-tried will have the same version # so the branch may already exist from the earlier attempt. 'checkout ' + base_branch, 'checkout {flag} {branch}'.format(flag=branch_flag, branch=head_branch), 'add ' + ' '.join([os.path.abspath(path) for path in files_added]), ] logging.debug('Commiting changes into local repository "%s" branch=%s', self.__docs_repository.git_dir, head_branch) git = self.__scm.git git.check_run_sequence(git_dir, local_git_commands) git.check_commit_or_no_changes(git_dir, '-m "{msg}"'.format(msg=message)) logging.info('Pushing branch="%s" into "%s"', head_branch, self.__docs_repository.origin) git.push_branch_to_origin(git_dir, branch=head_branch, force=True) def _prepare_local_repository_files(self): """Implements CommandProcessor interface.""" # And putting them into the SPINNAKER_GITHUB_IO_REPOSITORY_NAME # # NOTE(ewiseblatt): 20171218 # This is the current scheme, however I think this should read # os.path.join(git_dir, spinnaker_minor_branch, docs_path_in_repo) # where "spinnaker_minor_branch" is the version with the <PATCH> # replaced with 'x' target_path = os.path.join(self.__docs_repository.git_dir, 'reference', 'api', 'docs.html') logging.debug('Copying %s to %s', self.__html_path, target_path) shutil.copy2(self.__html_path, target_path) return [target_path]