def _ensure_templates_directory(self):
"""Initialize the templates_directory, pulling the repo if needed."""
scm = BranchSourceCodeManager(
make_options_with_fallback(self.options), self.get_input_dir()
)
repository = scm.make_repository_spec(SPINNAKER_IO_REPOSITORY_NAME)
# These documents arent tied to version control, especially since they are
# published to a different repository.
scm.ensure_git_path(repository)
self.__templates_directory = os.path.join(repository.git_dir, "_api_templates")
def test_build(self):
test_root = self.test_root
options = self.options
options.git_branch = PATCH_BRANCH
options.github_owner = 'default'
options.github_disable_upstream_push = True
scm = BranchSourceCodeManager(options, test_root)
golden_bom = dict(self.golden_bom)
builder = BomBuilder.new_from_bom(options, scm, golden_bom)
source_repositories = [
scm.make_repository_spec(name)
for name in ALL_STANDARD_TEST_BOM_REPO_NAMES
]
for repository in source_repositories:
scm.ensure_git_path(repository)
summary = scm.git.collect_repository_summary(repository.git_dir)
source_info = SourceInfo('SourceInfoBuildNumber', summary)
builder.add_repository(repository, source_info)
with patch('buildtool.bom_commands.now') as mock_now:
mock_now.return_value = datetime.datetime(2018, 1, 2, 3, 4, 5)
bom = builder.build()
golden_bom['version'] = 'patch-OptionBuildNumber'
golden_bom['timestamp'] = '2018-01-02 03:04:05'
golden_bom['services'][NORMAL_SERVICE]['version'] = (
PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber')
golden_bom['services'][OUTLIER_SERVICE]['version'] = (
PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber')
golden_bom['services']['monitoring-third-party']['version'] = (
PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber')
golden_bom['artifactSources'] = {
'debianRepository':
'https://dl.bintray.com/%s/%s' %
(options.bintray_org, options.bintray_debian_repository),
'dockerRegistry':
options.docker_registry,
'googleImageProject':
options.publish_gce_image_project,
'gitPrefix':
os.path.dirname(self.repo_commit_map[NORMAL_REPO]['ORIGIN'])
}
for key, value in bom['services'].items():
print '**** {0} -> {1}'.format(key, value)
print ' ==>> {0}'.format(golden_bom['services'][key])
self.assertEquals(value, golden_bom['services'][key])
for key, value in bom.items():
self.assertEquals(value, golden_bom[key])
self.assertEquals(golden_bom, bom)
def _ensure_templates_directory(self):
"""Initialize the templates_directory, pulling the repo if needed."""
scm = BranchSourceCodeManager(
make_options_with_fallback(self.options),
self.get_input_dir())
repository = scm.make_repository_spec(SPINNAKER_GITHUB_IO_REPOSITORY_NAME)
# These documents arent tied to version control, especially since they are
# published to a different repository.
scm.ensure_git_path(repository)
self.__templates_directory = os.path.join(
repository.git_dir, '_api_templates')
def test_rebuild(self):
test_root = self.test_root
options = self.options
options.git_branch = 'master'
options.github_owner = 'default'
options.github_disable_upstream_push = True
options.build_number = 'UpdatedBuildNumber'
scm = BranchSourceCodeManager(options, test_root)
builder = BomBuilder.new_from_bom(options, scm,
MetricsManager.singleton(),
self.golden_bom)
repository = scm.make_repository_spec(OUTLIER_REPO)
scm.ensure_git_path(repository)
scm.git.check_run(repository.git_dir, 'checkout ' + PATCH_BRANCH)
summary = scm.git.collect_repository_summary(repository.git_dir)
source_info = SourceInfo('SourceInfoBuildNumber', summary)
builder.add_repository(repository, source_info)
with patch('buildtool.bom_commands.now') as mock_now:
mock_now.return_value = datetime.datetime(2018, 1, 2, 3, 4, 5)
bom = builder.build()
updated_service = bom['services'][OUTLIER_SERVICE]
self.assertEqual(
updated_service, {
'commit': self.repo_commit_map[OUTLIER_REPO][PATCH_BRANCH],
'version': PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber'
})
# The bom should be the same as before, but with new timestamp/version
# and our service updated. And the artifactSources to our configs.
updated_bom = dict(self.golden_bom)
updated_bom['timestamp'] = '2018-01-02 03:04:05'
updated_bom['version'] = 'master-UpdatedBuildNumber'
updated_bom['services'][OUTLIER_SERVICE] = updated_service
updated_bom['artifactSources'] = {
'debianRepository':
'https://dl.bintray.com/%s/%s' %
(options.bintray_org, options.bintray_debian_repository),
'dockerRegistry':
options.docker_registry,
'googleImageProject':
options.publish_gce_image_project,
'gitPrefix':
self.golden_bom['artifactSources']['gitPrefix']
}
for key, value in updated_bom.items():
self.assertEqual(value, bom[key])
self.assertEqual(updated_bom, bom)
def test_rebuild(self):
test_root = self.test_root
options = self.options
options.git_branch = 'master'
options.github_owner = 'default'
options.github_disable_upstream_push = True
options.build_number = 'UpdatedBuildNumber'
scm = BranchSourceCodeManager(options, test_root)
builder = BomBuilder.new_from_bom(
options, scm, MetricsManager.singleton(), self.golden_bom)
repository = scm.make_repository_spec(OUTLIER_REPO)
scm.ensure_git_path(repository)
scm.git.check_run(repository.git_dir, 'checkout ' + PATCH_BRANCH)
summary = scm.git.collect_repository_summary(repository.git_dir)
source_info = SourceInfo('SourceInfoBuildNumber', summary)
builder.add_repository(repository, source_info)
with patch('buildtool.bom_commands.now') as mock_now:
mock_now.return_value = datetime.datetime(2018, 1, 2, 3, 4, 5)
bom = builder.build()
updated_service = bom['services'][OUTLIER_SERVICE]
self.assertEqual(updated_service, {
'commit': self.repo_commit_map[OUTLIER_REPO][PATCH_BRANCH],
'version': PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber'
})
# The bom should be the same as before, but with new timestamp/version
# and our service updated. And the artifactSources to our configs.
updated_bom = dict(self.golden_bom)
updated_bom['timestamp'] = '2018-01-02 03:04:05'
updated_bom['version'] = 'master-UpdatedBuildNumber'
updated_bom['services'][OUTLIER_SERVICE] = updated_service
updated_bom['artifactSources'] = {
'debianRepository': 'https://dl.bintray.com/%s/%s' % (
options.bintray_org, options.bintray_debian_repository),
'dockerRegistry': options.docker_registry,
'googleImageProject': options.publish_gce_image_project,
'gitPrefix': self.golden_bom['artifactSources']['gitPrefix']
}
for key, value in updated_bom.items():
self.assertEqual(value, bom[key])
self.assertEqual(updated_bom, bom)
def test_build(self):
test_root = self.test_root
options = self.options
options.git_branch = PATCH_BRANCH
options.github_owner = 'default'
options.github_disable_upstream_push = True
scm = BranchSourceCodeManager(options, test_root)
golden_bom = dict(self.golden_bom)
builder = BomBuilder.new_from_bom(
options, scm, MetricsManager.singleton(), golden_bom)
source_repositories = [scm.make_repository_spec(name)
for name in ALL_STANDARD_TEST_BOM_REPO_NAMES]
for repository in source_repositories:
scm.ensure_git_path(repository)
summary = scm.git.collect_repository_summary(repository.git_dir)
source_info = SourceInfo('SourceInfoBuildNumber', summary)
builder.add_repository(repository, source_info)
with patch('buildtool.bom_commands.now') as mock_now:
mock_now.return_value = datetime.datetime(2018, 1, 2, 3, 4, 5)
bom = builder.build()
golden_bom['version'] = 'patch-OptionBuildNumber'
golden_bom['timestamp'] = '2018-01-02 03:04:05'
golden_bom['services'][NORMAL_SERVICE]['version'] = (
PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber')
golden_bom['services'][OUTLIER_SERVICE]['version'] = (
PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber')
golden_bom['services']['monitoring-third-party']['version'] = (
PATCH_VERSION_NUMBER + '-SourceInfoBuildNumber')
golden_bom['artifactSources'] = {
'debianRepository': 'https://dl.bintray.com/%s/%s' % (
options.bintray_org, options.bintray_debian_repository),
'dockerRegistry': options.docker_registry,
'googleImageProject': options.publish_gce_image_project,
'gitPrefix': os.path.dirname(self.repo_commit_map[NORMAL_REPO]['ORIGIN'])
}
for key, value in bom['services'].items():
self.assertEqual(value, golden_bom['services'][key])
for key, value in bom.items():
self.assertEqual(value, golden_bom[key])
self.assertEqual(golden_bom, bom)
class PublishHalyardCommand(CommandProcessor):
"""Publish halyard version to the public repository."""
def __init__(self, factory, options, **kwargs):
options_copy = copy.copy(options)
options_copy.bom_path = None
options_copy.bom_version = None
options_copy.git_branch = 'master'
options_copy.github_hostname = 'github.com'
# Overrides later if --git_allow_publish_master_branch is false
super(PublishHalyardCommand, self).__init__(factory, options_copy,
**kwargs)
check_options_set(options, ['halyard_version'])
match = re.match(r'(\d+)\.(\d+)\.(\d+)-\d+', options.halyard_version)
if match is None:
raise_and_log_error(
ConfigError(
'--halyard_version={version} is not X.Y.Z-<buildnum>'.
format(version=options.halyard_version)))
self.__stable_version = '{major}.{minor}.{patch}'.format(
major=match.group(1), minor=match.group(2), patch=match.group(3))
self.__scm = BranchSourceCodeManager(options_copy,
self.get_input_dir())
self.__hal = HalRunner(options_copy)
self.__gradle = GradleRunner(options_copy, self.__scm, self.metrics)
self.__halyard_repo_md_path = os.path.join('docs', 'commands.md')
dash = self.options.halyard_version.find('-')
semver_str = self.options.halyard_version[0:dash]
semver_parts = semver_str.split('.')
if len(semver_parts) != 3:
raise_and_log_error(
ConfigError('Expected --halyard_version in the form X.Y.Z-N'))
self.__release_branch = 'release-{maj}.{min}.x'.format(
maj=semver_parts[0], min=semver_parts[1])
self.__release_tag = 'version-' + semver_str
self.__release_version = semver_str
def determine_halyard_commit(self):
"""Determine the commit_id that we want to publish."""
options = self.options
versions_url = options.halyard_version_commits_url
if not versions_url:
versions_url = '{base}/{filename}'.format(
base=options.halyard_bucket_base_url,
filename=BuildHalyardCommand.HALYARD_VERSIONS_BASENAME)
if os.path.exists(versions_url):
logging.debug('Loading halyard version info from file %s',
versions_url)
with open(versions_url, 'r') as stream:
version_data = stream.read()
else:
logging.debug('Loading halyard version info from bucket %s',
versions_url)
gsutil_output = check_subprocess(
'gsutil cat {url}'.format(url=versions_url),
stderr=subprocess.PIPE)
# The latest version of gsutil prints a bunch of python warnings to stdout
# (see b/152449160). This file is a series of lines that look like...
# 0.41.0-180209172926: 05f1e832ab438e5a980d1102e84cdb348a0ab055
# ...so we'll just throw out any lines that don't start with digits.
valid_lines = [
line for line in gsutil_output.splitlines()
if line[0].isdigit()
]
version_data = "\n".join(valid_lines)
commit = yaml.safe_load(version_data).get(options.halyard_version)
if commit is None:
raise_and_log_error(
ConfigError(
'Unknown halyard version "{version}" in "{url}"'.format(
version=options.halyard_version, url=versions_url)))
return commit
def _prepare_repository(self):
"""Prepare a local repository to build for release.
Were rebuilding it only to have nebula give a new distribution tag.
However we will also use the repository to tag and branch the release
into github so want to at least clone the repo regardless.
"""
logging.debug('Preparing repository for publishing a halyard release.')
commit = self.determine_halyard_commit()
repository = self.__scm.make_repository_spec(
SPINNAKER_HALYARD_REPOSITORY_NAME, commit_id=commit)
git_dir = repository.git_dir
if os.path.exists(git_dir):
logging.info('Deleting existing %s to build fresh.', git_dir)
shutil.rmtree(git_dir)
git = self.__scm.git
git.clone_repository_to_path(repository, commit=commit)
return repository
def _promote_halyard(self, repository):
"""Promote an existing build to become the halyard stable version."""
options = self.options
logfile = self.get_logfile_path('promote-all')
env = dict(os.environ)
env.update({
'PUBLISH_HALYARD_ARTIFACT_DOCKER_IMAGE_SRC_BASE':
options.halyard_artifact_registry_image_base,
'PUBLISH_HALYARD_BUCKET_BASE_URL':
options.halyard_bucket_base_url,
'PUBLISH_HALYARD_DOCKER_IMAGE_BASE':
options.halyard_docker_image_base
})
check_subprocesses_to_logfile(
'Promote Halyard',
logfile,
[
'gcloud docker -a', # if repo is private it needs authenticated
'./release/promote-all.sh {candidate} {stable}'.format(
candidate=options.halyard_version,
stable=self.__stable_version),
'./release/promote-all.sh {candidate} stable'.format(
candidate=options.halyard_version)
],
env=env,
cwd=repository.git_dir)
def _build_release(self, repository):
"""Rebuild the actual release debian package.
We dont necessarily need to rebuild here. We just need to push as
debian to the "-stable". However there isnt an easy way to do this.
Note that this is not the promoted version. For safety[*] and simplicity
we'll promote the candidate whose version was used to build this.
Ideally this function can go away.
[*] Safety because the candidate was tested whereas this build was not.
"""
# Ideally we would just modify the existing bintray version to add
# *-stable to the distributions, however it does not appear possible
# to patch the debian attributes of a bintray version, only the
# version metadata. Therefore, we'll rebuild it.
# Alternatively we could download the existing and push a new one,
# however I dont see how to get at the existing debian metadata and
# dont want to ommit something
options = self.options
git_dir = repository.git_dir
summary = self.__scm.git.collect_repository_summary(git_dir)
config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'cloudbuild', 'debs.yml')
substitutions = {
'_BRANCH_NAME': options.git_branch,
'_BRANCH_TAG': re.sub(r'\W', '_', options.git_branch),
'_BUILD_NUMBER': options.build_number,
'_IMAGE_NAME': 'halyard',
'_VERSION': summary.version
}
# Convert it to the format expected by gcloud: "_FOO=bar,_BAZ=qux"
substitutions_arg = ','.join('='.join((str(k), str(v)))
for k, v in substitutions.items())
command = ('gcloud builds submit '
' --account={account} --project={project}'
' --substitutions={substitutions_arg},'
' --config={config} .'.format(
account=options.gcb_service_account,
project=options.gcb_project,
substitutions_arg=substitutions_arg,
config=config_path))
logfile = self.get_logfile_path('build-deb')
check_subprocesses_to_logfile('building deb with published version',
logfile, [command],
cwd=git_dir)
def write_target_docs(self, source_repository, target_repository):
source_path = os.path.join(source_repository.git_dir,
self.__halyard_repo_md_path)
target_rel_path = os.path.join('reference', 'halyard', 'commands.md')
target_path = os.path.join(target_repository.git_dir, target_rel_path)
now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
logging.debug('Writing documentation into %s', target_path)
header = textwrap.dedent("""\
---
layout: single
title: "Commands"
sidebar:
nav: reference
---
Published: {now}
""".format(now=now))
with open(source_path, 'r') as source:
body = source.read()
with open(target_path, 'w') as stream:
stream.write(header)
stream.write(body)
return target_rel_path
def push_docs(self, repository):
base_branch = 'master'
target_repository = self.__scm.make_repository_spec(
SPINNAKER_GITHUB_IO_REPOSITORY_NAME)
self.__scm.ensure_git_path(target_repository)
target_rel_path = self.write_target_docs(repository, target_repository)
if self.options.git_allow_publish_master_branch:
head_branch = 'master'
branch_flag = ''
else:
head_branch = self.__release_version + '-haldocs'
branch_flag = '-b'
logging.debug('Commiting changes into local repository "%s" branch=%s',
target_repository.git_dir, head_branch)
git_dir = target_repository.git_dir
message = 'docs(halyard): ' + self.__release_version
local_git_commands = [
# These commands are accomodating to a branch already existing
# because the branch is on the version, not build. A rejected
# build for some reason that is re-tried will have the same version
# so the branch may already exist from the earlier attempt.
'checkout ' + base_branch,
'checkout {flag} {branch}'.format(flag=branch_flag,
branch=head_branch),
'add ' + target_rel_path,
]
logging.debug('Commiting changes into local repository "%s" branch=%s',
target_repository.git_dir, head_branch)
git = self.__scm.git
git.check_run_sequence(git_dir, local_git_commands)
git.check_commit_or_no_changes(
git_dir, '-m "{msg}" {path}'.format(msg=message,
path=target_rel_path))
logging.info('Pushing halyard docs to %s branch="%s"',
target_repository.origin, head_branch)
git.push_branch_to_origin(target_repository.git_dir,
branch=head_branch,
force=True)
def _do_command(self):
"""Implements CommandProcessor interface."""
repository = self._prepare_repository()
self._build_release(repository)
self._promote_halyard(repository)
build_halyard_docs(self, repository)
self.push_docs(repository)
self.push_tag_and_branch(repository)
self.__hal.publish_halyard_release(self.__release_version)
def push_tag_and_branch(self, repository):
"""Pushes a stable branch and git version tag to the origin repository."""
git_dir = repository.git_dir
git = self.__scm.git
release_url = git.determine_push_url(repository.origin)
logging.info('Pushing branch=%s and tag=%s to %s',
self.__release_branch, self.__release_tag, release_url)
existing_commit = git.query_commit_at_tag(git_dir, self.__release_tag)
if existing_commit:
want_commit = git.query_local_repository_commit_id(git_dir)
if want_commit == existing_commit:
logging.debug('Already have "%s" at %s', self.__release_tag,
want_commit)
return
git.check_run_sequence(git_dir, [
'checkout -b ' + self.__release_branch, 'remote add release ' +
release_url, 'push release ' + self.__release_branch,
'tag ' + self.__release_tag, 'push release ' + self.__release_tag
])
class PublishHalyardCommand(CommandProcessor):
"""Publish halyard version to the public repository."""
def __init__(self, factory, options, **kwargs):
options_copy = copy.copy(options)
options_copy.bom_path = None
options_copy.bom_version = None
options_copy.git_branch = 'master'
options_copy.github_hostname = 'github.com'
# Overrides later if --git_allow_publish_master_branch is false
super(PublishHalyardCommand, self).__init__(factory, options_copy, **kwargs)
check_options_set(options, ['halyard_version'])
match = re.match(r'(\d+)\.(\d+)\.(\d+)-\d+', options.halyard_version)
if match is None:
raise_and_log_error(
ConfigError('--halyard_version={version} is not X.Y.Z-<buildnum>'
.format(version=options.halyard_version)))
self.__stable_version = '{major}.{minor}.{patch}'.format(
major=match.group(1), minor=match.group(2), patch=match.group(3))
self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir())
self.__hal = HalRunner(options_copy)
self.__gradle = GradleRunner(options_copy, self.__scm, self.metrics)
self.__halyard_repo_md_path = os.path.join('docs', 'commands.md')
dash = self.options.halyard_version.find('-')
semver_str = self.options.halyard_version[0:dash]
semver_parts = semver_str.split('.')
if len(semver_parts) != 3:
raise_and_log_error(
ConfigError('Expected --halyard_version in the form X.Y.Z-N'))
self.__release_branch = 'release-{maj}.{min}.x'.format(
maj=semver_parts[0], min=semver_parts[1])
self.__release_tag = 'version-' + semver_str
self.__release_version = semver_str
def determine_halyard_commit(self):
"""Determine the commit_id that we want to publish."""
options = self.options
versions_url = options.halyard_version_commits_url
if not versions_url:
versions_url = '{base}/{filename}'.format(
base=options.halyard_bucket_base_url,
filename=BuildHalyardCommand.HALYARD_VERSIONS_BASENAME)
if os.path.exists(versions_url):
logging.debug('Loading halyard version info from file %s', versions_url)
with open(versions_url, 'r') as stream:
version_data = stream.read()
else:
logging.debug('Loading halyard version info from bucket %s', versions_url)
version_data = check_subprocess(
'gsutil cat {url}'.format(url=versions_url))
commit = yaml.safe_load(version_data).get(options.halyard_version)
if commit is None:
raise_and_log_error(
ConfigError('Unknown halyard version "{version}" in "{url}"'.format(
version=options.halyard_version, url=versions_url)))
return commit
def _prepare_repository(self):
"""Prepare a local repository to build for release.
Were rebuilding it only to have nebula give a new distribution tag.
However we will also use the repository to tag and branch the release
into github so want to at least clone the repo regardless.
"""
logging.debug('Preparing repository for publishing a halyard release.')
commit = self.determine_halyard_commit()
repository = self.__scm.make_repository_spec(
SPINNAKER_HALYARD_REPOSITORY_NAME, commit_id=commit)
git_dir = repository.git_dir
if os.path.exists(git_dir):
logging.info('Deleting existing %s to build fresh.', git_dir)
shutil.rmtree(git_dir)
git = self.__scm.git
git.clone_repository_to_path(repository, commit=commit)
return repository
def _promote_halyard(self, repository):
"""Promote an existing build to become the halyard stable version."""
options = self.options
logfile = self.get_logfile_path('promote-all')
env = dict(os.environ)
env.update({
'PUBLISH_HALYARD_BUCKET_BASE_URL': options.halyard_bucket_base_url,
'PUBLISH_HALYARD_DOCKER_IMAGE_BASE': options.halyard_docker_image_base
})
check_subprocesses_to_logfile(
'Promote Halyard', logfile,
['gcloud docker -a', # if repo is private it needs authenticated
'./release/promote-all.sh {candidate} {stable}'.format(
candidate=options.halyard_version,
stable=self.__stable_version),
'./release/promote-all.sh {candidate} stable'.format(
candidate=options.halyard_version)],
env=env,
cwd=repository.git_dir)
def _build_release(self, repository):
"""Rebuild the actual release debian package.
We dont necessarily need to rebuild here. We just need to push as
debian to the "-stable". However there isnt an easy way to do this.
Note that this is not the promoted version. For safety[*] and simplicity
we'll promote the candidate whose version was used to build this.
Ideally this function can go away.
[*] Safety because the candidate was tested whereas this build was not.
"""
# Ideally we would just modify the existing bintray version to add
# *-stable to the distributions, however it does not appear possible
# to patch the debian attributes of a bintray version, only the
# version metadata. Therefore, we'll rebuild it.
# Alternatively we could download the existing and push a new one,
# however I dont see how to get at the existing debian metadata and
# dont want to ommit something
git_dir = repository.git_dir
summary = self.__scm.git.collect_repository_summary(git_dir)
args = self.__gradle.get_common_args()
args.extend(self.__gradle.get_debian_args(
'trusty-stable,xenial-stable,bionic-stable'))
build_number = self.options.build_number
self.__gradle.check_run(
args, self, repository, 'candidate', 'build-release',
version=self.__release_version, build_number=build_number,
gradle_dir=git_dir)
info_path = os.path.join(self.get_output_dir(), 'halyard_info.yml')
logging.debug('Writing build information to %s', info_path)
write_to_path(summary.to_yaml(), info_path)
def write_target_docs(self, source_repository, target_repository):
source_path = os.path.join(source_repository.git_dir,
self.__halyard_repo_md_path)
target_rel_path = os.path.join('reference', 'halyard', 'commands.md')
target_path = os.path.join(target_repository.git_dir, target_rel_path)
now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
logging.debug('Writing documentation into %s', target_path)
header = textwrap.dedent(
"""\
---
layout: single
title: "Commands"
sidebar:
nav: reference
---
Published: {now}
""".format(now=now))
with open(source_path, 'r') as source:
body = source.read()
with open(target_path, 'w') as stream:
stream.write(header)
stream.write(body)
return target_rel_path
def push_docs(self, repository):
base_branch = 'master'
target_repository = self.__scm.make_repository_spec(
SPINNAKER_GITHUB_IO_REPOSITORY_NAME)
self.__scm.ensure_git_path(target_repository)
target_rel_path = self.write_target_docs(repository, target_repository)
if self.options.git_allow_publish_master_branch:
head_branch = 'master'
branch_flag = ''
else:
head_branch = self.__release_version + '-haldocs'
branch_flag = '-b'
logging.debug('Commiting changes into local repository "%s" branch=%s',
target_repository.git_dir, head_branch)
git_dir = target_repository.git_dir
message = 'docs(halyard): ' + self.__release_version
local_git_commands = [
# These commands are accomodating to a branch already existing
# because the branch is on the version, not build. A rejected
# build for some reason that is re-tried will have the same version
# so the branch may already exist from the earlier attempt.
'checkout ' + base_branch,
'checkout {flag} {branch}'.format(
flag=branch_flag, branch=head_branch),
'add ' + target_rel_path,
]
logging.debug('Commiting changes into local repository "%s" branch=%s',
target_repository.git_dir, head_branch)
git = self.__scm.git
git.check_run_sequence(git_dir, local_git_commands)
git.check_commit_or_no_changes(
git_dir, '-m "{msg}" {path}'.format(msg=message, path=target_rel_path))
logging.info('Pushing halyard docs to %s branch="%s"',
target_repository.origin, head_branch)
git.push_branch_to_origin(
target_repository.git_dir, branch=head_branch, force=True)
def _do_command(self):
"""Implements CommandProcessor interface."""
repository = self._prepare_repository()
self._build_release(repository)
self._promote_halyard(repository)
build_halyard_docs(self, repository)
self.push_docs(repository)
self.push_tag_and_branch(repository)
self.__hal.publish_halyard_release(self.__release_version)
def push_tag_and_branch(self, repository):
"""Pushes a stable branch and git version tag to the origin repository."""
git_dir = repository.git_dir
git = self.__scm.git
release_url = git.determine_push_url(repository)
logging.info('Pushing branch=%s and tag=%s to %s',
self.__release_branch, self.__release_tag, release_url)
existing_commit = git.query_commit_at_tag(git_dir, self.__release_tag)
if existing_commit:
want_commit = git.query_local_repository_commit_id(git_dir)
if want_commit == existing_commit:
logging.debug('Already have "%s" at %s',
self.__release_tag, want_commit)
return
git.check_run_sequence(
git_dir,
[
'checkout -b ' + self.__release_branch,
'remote add release ' + release_url,
'push release ' + self.__release_branch,
'tag ' + self.__release_tag,
'push release ' + self.__release_tag
])
class PublishHalyardCommand(CommandProcessor):
"""Publish halyard version to the public repository."""
def __init__(self, factory, options, **kwargs):
options_copy = copy.copy(options)
options_copy.bom_path = None
options_copy.bom_version = None
options_copy.git_branch = 'master'
# Overrides later if --git_allow_publish_master_branch is false
super(PublishHalyardCommand, self).__init__(factory, options_copy,
**kwargs)
self.__scm = BranchSourceCodeManager(options_copy,
self.get_input_dir())
self.__hal = HalRunner(options_copy)
self.__gradle = GradleRunner(options_copy, self.__scm, self.metrics)
self.__halyard_repo_md_path = os.path.join('docs', 'commands.md')
dash = self.options.halyard_version.find('-')
semver_str = self.options.halyard_version[0:dash]
semver_parts = semver_str.split('.')
if len(semver_parts) != 3:
raise_and_log_error(
ConfigError('Expected --halyard_version in the form X.Y.Z-N'))
self.__release_branch = 'release-{maj}.{min}.x'.format(
maj=semver_parts[0], min=semver_parts[1])
self.__release_tag = 'version-' + semver_str
self.__release_version = semver_str
def determine_commit(self, repository):
"""Determine the commit_id that we want to publish."""
if repository.name != 'halyard':
raise_and_log_error(
ConfigError('Unexpected repository "%s"' % repository.name))
options = self.options
versions_url = options.halyard_version_commits_url
if not versions_url:
versions_url = '{base}/{filename}'.format(
base=options.halyard_bucket_base_url,
filename=BuildHalyardCommand.HALYARD_VERSIONS_BASENAME)
if os.path.exists(versions_url):
logging.debug('Loading halyard version info from file %s',
versions_url)
with open(versions_url, 'r') as stream:
version_data = stream.read()
else:
logging.debug('Loading halyard version info from bucket %s',
versions_url)
version_data = check_subprocess(
'gsutil cat {url}'.format(url=versions_url))
commit = yaml.load(version_data).get(options.halyard_version)
if commit is None:
raise_and_log_error(
ConfigError(
'Unknown halyard version "{version}" in "{url}"'.format(
version=options.halyard_version, url=versions_url)))
def _prepare_repository(self):
"""Prepare a local repository to build for release.
Were rebuilding it only to have nebula give a new distribution tag.
However we will also use the repository to tag and branch the release
into github so want to at least clone the repo regardless.
"""
logging.debug('Preparing repository for publishing a halyard release.')
repository = self.__scm.make_repository_spec(
SPINNAKER_HALYARD_REPOSITORY_NAME)
commit = self.determine_commit(repository)
git_dir = repository.git_dir
if os.path.exists(git_dir):
logging.info('Deleting existing %s to build fresh.', git_dir)
shutil.rmtree(git_dir)
git = self.__scm.git
git.clone_repository_to_path(repository, commit=commit)
self.__scm.refresh_source_info(repository, self.options.build_number)
return repository
def _build_release(self, repository):
"""Rebuild the actual release.
We dont necessarily need to rebuild here. We just need to push as
debian to the "-stable".
"""
# Ideally we would just modify the existing bintray version to add
# trusty-stable to the distributions, however it does not appear possible
# to patch the debian attributes of a bintray version, only the
# version metadata. Therefore, we'll rebuild it.
# Alternatively we could download the existing and push a new one,
# however I dont see how to get at the existing debian metadata and
# dont want to ommit something
git_dir = repository.git_dir
summary = self.__scm.git.collect_repository_summary(git_dir)
args = self.__gradle.get_common_args()
args.extend(
self.__gradle.get_debian_args('trusty-stable,xenial-stable'))
build_number = self.options.build_number
if not self.__gradle.consider_debian_on_bintray(
repository, build_number=build_number):
self.__gradle.check_run(args,
self,
repository,
'candidate',
'build-release',
version=self.__release_version,
build_number=build_number,
gradle_dir=git_dir)
info_path = os.path.join(self.get_output_dir(), 'halyard_info.yml')
logging.debug('Writing build information to %s', info_path)
write_to_path(summary.to_yaml(), info_path)
def write_target_docs(self, source_repository, target_repository):
source_path = os.path.join(source_repository.git_dir,
self.__halyard_repo_md_path)
target_rel_path = os.path.join('reference', 'halyard', 'commands.md')
target_path = os.path.join(target_repository.git_dir, target_rel_path)
now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
logging.debug('Writing documentation into %s', target_path)
header = textwrap.dedent("""\
---
layout: single
title: "Commands"
sidebar:
nav: reference
---
Published: {now}
""".format(now=now))
with open(source_path, 'r') as source:
body = source.read()
with open(target_path, 'w') as stream:
stream.write(header)
stream.write(body)
return target_rel_path
def push_docs(self, repository):
base_branch = 'master'
target_repository = self.__scm.make_repository_spec(
SPINNAKER_GITHUB_IO_REPOSITORY_NAME)
self.__scm.ensure_git_path(target_repository)
target_rel_path = self.write_target_docs(repository, target_repository)
if self.options.git_allow_publish_master_branch:
head_branch = 'master'
branch_flag = ''
else:
head_branch = self.__release_version + '-haldocs'
branch_flag = '-b'
logging.debug('Commiting changes into local repository "%s" branch=%s',
target_repository.git_dir, head_branch)
git_dir = target_repository.git_dir
message = 'docs(halyard): ' + self.__release_version
local_git_commands = [
# These commands are accomodating to a branch already existing
# because the branch is on the version, not build. A rejected
# build for some reason that is re-tried will have the same version
# so the branch may already exist from the earlier attempt.
'checkout ' + base_branch,
'checkout {flag} {branch}'.format(flag=branch_flag,
branch=head_branch),
'add ' + target_rel_path,
'commit -m "{msg}" {path}'.format(msg=message,
path=target_rel_path),
]
logging.debug('Commiting changes into local repository "%s" branch=%s',
target_repository.git_dir, head_branch)
git = self.__scm.git
git.check_run_sequence(git_dir, local_git_commands)
logging.info('Pushing halyard docs to %s branch="%s"',
target_repository.origin, head_branch)
git.push_branch_to_origin(target_repository.git_dir,
branch=head_branch)
def _do_command(self):
"""Implements CommandProcessor interface."""
repository = self._prepare_repository()
self._build_release(repository)
build_halyard_docs(self, repository)
self.push_docs(repository)
self.push_tag_and_branch(repository)
self.__hal.publish_halyard_release(self.__release_version)
def push_tag_and_branch(self, repository):
"""Pushes a stable branch and git version tag to the origin repository."""
git_dir = repository.git_dir
git = self.__scm.git
release_url = repository.origin
logging.info('Pushing branch=%s and tag=%s to %s', self.__release_tag,
self.__release_branch, release_url)
git.check_run_sequence(git_dir, [
'checkout -b ' + self.__release_branch, 'remote add release ' +
release_url, 'push release ' + self.__release_branch,
'tag ' + self.__release_tag, 'push release ' + self.__release_tag
])
class PublishApiDocsCommand(CommandProcessor):
"""Publish built docs back up to the origin repository.
This uses git_runner to checkout the SPINNAKER_GITHUB_IO_REPOSITORY
that we'll be publishing docs to. It assumes that the documentation has
already been built so that it could have been reviewed.
"""
def __init__(self, factory, options, **kwargs):
super(PublishApiDocsCommand, self).__init__(factory, options, **kwargs)
docs_dir = self.get_output_dir(command=BUILD_DOCS_COMMAND)
self.__html_path = os.path.join(os.path.abspath(docs_dir),
'index.html')
self._check_args()
self.__scm = BranchSourceCodeManager(
make_options_with_fallback(self.options), self.get_input_dir())
self.__docs_repository = self.__scm.make_repository_spec(
SPINNAKER_GITHUB_IO_REPOSITORY_NAME)
def _check_args(self):
check_path_exists(self.__html_path,
why='output from running "build_apidocs"')
check_options_set(self.options, ['spinnaker_version'])
def _do_command(self):
"""Implements CommandProcessor interface."""
self.__scm.ensure_git_path(self.__docs_repository, branch='master')
base_branch = 'master'
version = self.options.spinnaker_version
if self.options.git_allow_publish_master_branch:
head_branch = 'master'
branch_flag = ''
else:
head_branch = version + '-apidocs'
branch_flag = '-b'
files_added = self._prepare_local_repository_files()
git_dir = self.__docs_repository.git_dir
message = 'docs(api): API Documentation for Spinnaker ' + version
local_git_commands = [
# These commands are accomodating to a branch already existing
# because the branch is on the version, not build. A rejected
# build for some reason that is re-tried will have the same version
# so the branch may already exist from the earlier attempt.
'checkout ' + base_branch,
'checkout {flag} {branch}'.format(flag=branch_flag,
branch=head_branch),
'add ' + ' '.join([os.path.abspath(path) for path in files_added]),
]
logging.debug('Commiting changes into local repository "%s" branch=%s',
self.__docs_repository.git_dir, head_branch)
git = self.__scm.git
git.check_run_sequence(git_dir, local_git_commands)
git.check_commit_or_no_changes(git_dir,
'-m "{msg}"'.format(msg=message))
logging.info('Pushing branch="%s" into "%s"', head_branch,
self.__docs_repository.origin)
git.push_branch_to_origin(git_dir, branch=head_branch, force=True)
def _prepare_local_repository_files(self):
"""Implements CommandProcessor interface."""
# And putting them into the SPINNAKER_GITHUB_IO_REPOSITORY_NAME
#
# NOTE(ewiseblatt): 20171218
# This is the current scheme, however I think this should read
# os.path.join(git_dir, spinnaker_minor_branch, docs_path_in_repo)
# where "spinnaker_minor_branch" is the version with the <PATCH>
# replaced with 'x'
target_path = os.path.join(self.__docs_repository.git_dir, 'reference',
'api', 'docs.html')
logging.debug('Copying %s to %s', self.__html_path, target_path)
shutil.copy2(self.__html_path, target_path)
return [target_path]
class PublishApiDocsCommand(CommandProcessor):
"""Publish built docs back up to the origin repository.
This uses git_runner to checkout the SPINNAKER_GITHUB_IO_REPOSITORY
that we'll be publishing docs to. It assumes that the documentation has
already been built so that it could have been reviewed.
"""
def __init__(self, factory, options, **kwargs):
super(PublishApiDocsCommand, self).__init__(
factory, options, **kwargs)
docs_dir = self.get_output_dir(command=BUILD_DOCS_COMMAND)
self.__html_path = os.path.join(os.path.abspath(docs_dir), 'index.html')
self._check_args()
self.__scm = BranchSourceCodeManager(
make_options_with_fallback(self.options),
self.get_input_dir())
self.__docs_repository = self.__scm.make_repository_spec(
SPINNAKER_GITHUB_IO_REPOSITORY_NAME)
def _check_args(self):
check_path_exists(self.__html_path,
why='output from running "build_apidocs"')
check_options_set(self.options, ['spinnaker_version'])
def _do_command(self):
"""Implements CommandProcessor interface."""
self.__scm.ensure_git_path(self.__docs_repository, branch='master')
base_branch = 'master'
version = self.options.spinnaker_version
if self.options.git_allow_publish_master_branch:
head_branch = 'master'
branch_flag = ''
else:
head_branch = version + '-apidocs'
branch_flag = '-b'
files_added = self._prepare_local_repository_files()
git_dir = self.__docs_repository.git_dir
message = 'docs(api): API Documentation for Spinnaker ' + version
local_git_commands = [
# These commands are accomodating to a branch already existing
# because the branch is on the version, not build. A rejected
# build for some reason that is re-tried will have the same version
# so the branch may already exist from the earlier attempt.
'checkout ' + base_branch,
'checkout {flag} {branch}'.format(
flag=branch_flag, branch=head_branch),
'add ' + ' '.join([os.path.abspath(path) for path in files_added]),
]
logging.debug('Commiting changes into local repository "%s" branch=%s',
self.__docs_repository.git_dir, head_branch)
git = self.__scm.git
git.check_run_sequence(git_dir, local_git_commands)
git.check_commit_or_no_changes(git_dir, '-m "{msg}"'.format(msg=message))
logging.info('Pushing branch="%s" into "%s"',
head_branch, self.__docs_repository.origin)
git.push_branch_to_origin(git_dir, branch=head_branch, force=True)
def _prepare_local_repository_files(self):
"""Implements CommandProcessor interface."""
# And putting them into the SPINNAKER_GITHUB_IO_REPOSITORY_NAME
#
# NOTE(ewiseblatt): 20171218
# This is the current scheme, however I think this should read
# os.path.join(git_dir, spinnaker_minor_branch, docs_path_in_repo)
# where "spinnaker_minor_branch" is the version with the <PATCH>
# replaced with 'x'
target_path = os.path.join(
self.__docs_repository.git_dir, 'reference', 'api', 'docs.html')
logging.debug('Copying %s to %s', self.__html_path, target_path)
shutil.copy2(self.__html_path, target_path)
return [target_path]
def test_rebuild(self):
test_root = self.test_root
options = self.options
options.git_branch = "master"
options.github_owner = "default"
options.github_disable_upstream_push = True
options.build_number = "UpdatedBuildNumber"
scm = BranchSourceCodeManager(options, test_root)
builder = BomBuilder.new_from_bom(options, scm,
MetricsManager.singleton(),
self.golden_bom)
repository = scm.make_repository_spec(OUTLIER_REPO)
scm.ensure_git_path(repository)
scm.git.check_run(repository.git_dir, "checkout " + PATCH_BRANCH)
summary = scm.git.collect_repository_summary(repository.git_dir)
source_info = SourceInfo("SourceInfoBuildNumber", summary)
builder.add_repository(repository, source_info)
with patch("buildtool.bom_commands.now") as mock_now:
mock_now.return_value = datetime.datetime(2018, 1, 2, 3, 4, 5)
bom = builder.build()
updated_service = bom["services"][OUTLIER_SERVICE]
# OUTLIER_REPO hasn't been tagged since extra commits so bom should be same
self.assertNotEqual(
updated_service,
{
"commit": self.repo_commit_map[OUTLIER_REPO][PATCH_BRANCH],
"version": BASE_VERSION_NUMBER,
},
)
# The bom should be the same as before, but with new timestamp/version
# and our service unchanged. And the artifactSources to our configs.
updated_bom = dict(self.golden_bom)
updated_bom["timestamp"] = "2018-01-02 03:04:05"
updated_bom["version"] = "UpdatedBuildNumber"
updated_bom["artifactSources"] = {
"gitPrefix": self.golden_bom["artifactSources"]["gitPrefix"],
"debianRepository": SPINNAKER_DEBIAN_REPOSITORY,
"dockerRegistry": SPINNAKER_DOCKER_REGISTRY,
"googleImageProject": SPINNAKER_GOOGLE_IMAGE_PROJECT,
}
for key, value in bom["services"].items():
# monitoring-daemon has extra commit on branch so commit id's should not match
if key in ["monitoring-daemon", "monitoring-third-party"]:
self.assertNotEqual(
value,
updated_bom["services"][key],
msg="key: {} - value: {}".format(key, value),
)
else:
self.assertEqual(
value,
updated_bom["services"][key],
msg="key: {} - value: {}".format(key, value),
)
for key, value in bom.items():
if key != "services":
self.assertEqual(value, updated_bom[key])
def test_build(self):
test_root = self.test_root
options = self.options
options.git_branch = PATCH_BRANCH
options.github_owner = "default"
options.github_disable_upstream_push = True
scm = BranchSourceCodeManager(options, test_root)
golden_bom = dict(self.golden_bom)
builder = BomBuilder.new_from_bom(options, scm,
MetricsManager.singleton(),
golden_bom)
source_repositories = [
scm.make_repository_spec(name)
for name in ALL_STANDARD_TEST_BOM_REPO_NAMES
]
for repository in source_repositories:
scm.ensure_git_path(repository)
summary = scm.git.collect_repository_summary(repository.git_dir)
source_info = SourceInfo("SourceInfoBuildNumber", summary)
builder.add_repository(repository, source_info)
with patch("buildtool.bom_commands.now") as mock_now:
mock_now.return_value = datetime.datetime(2018, 1, 2, 3, 4, 5)
bom = builder.build()
golden_bom["version"] = "OptionBuildNumber"
golden_bom["timestamp"] = "2018-01-02 03:04:05"
golden_bom["services"][NORMAL_SERVICE]["version"] = BASE_VERSION_NUMBER
golden_bom["services"][OUTLIER_SERVICE][
"version"] = BASE_VERSION_NUMBER
golden_bom["services"]["monitoring-third-party"][
"version"] = BASE_VERSION_NUMBER
golden_bom["artifactSources"] = {
"gitPrefix":
os.path.dirname(self.repo_commit_map[NORMAL_REPO]["ORIGIN"]),
"debianRepository":
SPINNAKER_DEBIAN_REPOSITORY,
"dockerRegistry":
SPINNAKER_DOCKER_REGISTRY,
"googleImageProject":
SPINNAKER_GOOGLE_IMAGE_PROJECT,
}
for key, value in bom["services"].items():
# gate has extra commit on branch so commit id's should not match
if key in ["gate", "monitoring-daemon", "monitoring-third-party"]:
self.assertNotEqual(
value,
golden_bom["services"][key],
msg="key: {} - value: {}".format(key, value),
)
else:
self.assertEqual(
value,
golden_bom["services"][key],
msg="key: {} - value: {}".format(key, value),
)
for key, value in bom.items():
if key != "services":
self.assertEqual(value, golden_bom[key])
