def build_all_halyard_deployments(self, repository):
"""Helper function for building halyard."""
options = self.options
git_dir = repository.git_dir
summary = self.source_code_manager.git.collect_repository_summary(git_dir)
self.__build_version = '%s-%s' % (summary.version, options.build_number)
cmd = './release/all.sh {version} nightly'.format(
version=self.__build_version)
env = dict(os.environ)
env.update({
'PUBLISH_HALYARD_BUCKET_BASE_URL': options.halyard_bucket_base_url,
'PUBLISH_HALYARD_DOCKER_IMAGE_BASE': options.halyard_docker_image_base
})
logging.info(
'Preparing the environment variables for release/all.sh:\n'
' PUBLISH_HALYARD_DOCKER_IMAGE_BASE=%s\n'
' PUBLISH_HALYARD_BUCKET_BASE_URL=%s',
options.halyard_docker_image_base,
options.halyard_bucket_base_url)
logfile = self.get_logfile_path('jar-build')
check_subprocesses_to_logfile(
'{name} build'.format(name='halyard'), logfile,
[cmd], cwd=git_dir, env=env)
def build_halyard_docs(command, repository):
"""Builds Halyard's CLI and updates documentation in its repo."""
cli_dir = os.path.join(repository.git_dir, 'halyard-cli')
# Before, we were doing this first:
# check_run_quick('git -C halyard rev-parse HEAD'
# ' | xargs git -C halyard checkout ;')
# however now the repository should already be at the desired commit.
logging.debug('Building Halyard CLI and docs.')
logfile = command.get_logfile_path('build-docs')
check_subprocesses_to_logfile(
'Build halyard docs', logfile, ['make'], cwd=cli_dir)
def test_check_to_file_subprocess_failed(self):
cmd = '/bin/ls /abc/def'
path = os.path.join(self.base_temp_dir, 'check_failed.log')
with self.assertRaises(ExecutionError) as ex:
check_subprocesses_to_logfile('Test Logfile', path, [cmd])
self.assertTrue(hasattr(ex.exception, 'loggedit'))
self.assertTrue(os.path.exists(path))
with open(path, 'r') as stream:
lines = stream.read().split('\n')
body = '\n'.join(lines[3:-3]).strip()
expect = "/bin/ls: cannot access '/abc/def': No such file or directory"
self.assertEqual(expect, body)
def build_halyard_docs(command, repository):
"""Builds Halyard's CLI and updates documentation in its repo."""
cli_dir = os.path.join(repository.git_dir, 'halyard-cli')
# Before, we were doing this first:
# check_run_quick('git -C halyard rev-parse HEAD'
# ' | xargs git -C halyard checkout ;')
# however now the repository should already be at the desired commit.
logging.debug('Building Halyard CLI and docs.')
logfile = command.get_logfile_path('build-docs')
check_subprocesses_to_logfile(
'Build halyard docs', logfile, ['make'], cwd=cli_dir)
def test_check_to_file_subprocess_failed(self):
cmd = "/bin/ls /abc/def"
path = os.path.join(self.base_temp_dir, "check_failed.log")
with self.assertRaises(ExecutionError) as ex:
check_subprocesses_to_logfile("Test Logfile", path, [cmd])
self.assertTrue(hasattr(ex.exception, "loggedit"))
self.assertTrue(os.path.exists(path))
with open(path, "r") as stream:
lines = stream.read().split("\n")
body = "\n".join(lines[3:-3]).strip()
expect = "/bin/ls: cannot access '/abc/def': No such file or directory"
self.assertEqual(expect, body)
def run_and_log_test_script(command):
logfile = os.path.join(self.options.output_dir, 'citest_logs',
'%s-%s.console.log' % (test_name, os.getpid()))
logging.info('Logging test "%s" to %s', test_name, logfile)
try:
check_subprocesses_to_logfile('running test', logfile, [command])
retcode = 0
logging.info('Test %s PASSED -- see %s', test_name, logfile)
except:
retcode = -1
logging.info('Test %s FAILED -- see %s', test_name, logfile)
return retcode, logfile
def _build_release(self, repository):
"""Rebuild the actual release debian package.
We dont necessarily need to rebuild here. We just need to push as
debian to the "-stable". However there isnt an easy way to do this.
Note that this is not the promoted version. For safety[*] and simplicity
we'll promote the candidate whose version was used to build this.
Ideally this function can go away.
[*] Safety because the candidate was tested whereas this build was not.
"""
# Ideally we would just modify the existing bintray version to add
# *-stable to the distributions, however it does not appear possible
# to patch the debian attributes of a bintray version, only the
# version metadata. Therefore, we'll rebuild it.
# Alternatively we could download the existing and push a new one,
# however I dont see how to get at the existing debian metadata and
# dont want to ommit something
options = self.options
git_dir = repository.git_dir
summary = self.__scm.git.collect_repository_summary(git_dir)
config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"cloudbuild", "debs.yml")
substitutions = {
"_BRANCH_NAME": options.git_branch,
"_BRANCH_TAG": re.sub(r"\W", "_", options.git_branch),
"_BUILD_NUMBER": options.build_number,
"_IMAGE_NAME": "halyard",
"_VERSION": summary.version,
}
# Convert it to the format expected by gcloud: "_FOO=bar,_BAZ=qux"
substitutions_arg = ",".join("=".join((str(k), str(v)))
for k, v in substitutions.items())
command = ("gcloud builds submit "
" --account={account} --project={project}"
" --substitutions={substitutions_arg},"
" --config={config} .".format(
account=options.gcb_service_account,
project=options.gcb_project,
substitutions_arg=substitutions_arg,
config=config_path,
))
logfile = self.get_logfile_path("build-deb")
check_subprocesses_to_logfile("building deb with published version",
logfile, [command],
cwd=git_dir)
def _do_repository(self, repository):
"""Implements RepositoryCommandProcessor interface."""
name = repository.name
build_component_image_sh = os.path.join(
os.path.dirname(__file__), '..', 'build_google_component_image.sh')
options = self.options
bom_version = self.source_code_manager.determine_bom_version()
command_line = [
build_component_image_sh, '--artifact ', name, '--account',
options.build_gce_service_account, '--hal_daemon_endpoint',
'http://' + options.halyard_daemon, '--build_project',
options.build_gce_project, '--install_script',
options.install_image_script, '--publish_project',
self.__image_project, '--publish_script',
options.publish_gce_image_script, '--version', bom_version,
'--zone', options.build_gce_zone
]
command_line.extend(self.__determine_repo_install_args(repository))
extra_install_args = []
if options.halyard_bom_bucket:
extra_install_args.extend(
['--halyard_config_bucket', options.halyard_bom_bucket])
if options.bintray_debian_repository:
bintray_url = 'https://dl.bintray.com/{org}/{repo}'.format(
org=options.bintray_org,
repo=options.bintray_debian_repository)
extra_install_args.extend([
'--release_track', options.halyard_release_track,
'--halyard_repository', bintray_url, '--spinnaker_repository',
bintray_url
])
if extra_install_args:
command_line.extend([
'--extra_install_script_args',
'"{0}"'.format(' '.join(extra_install_args))
])
command = ' '.join(command_line)
logfile = self.get_logfile_path(name + '-gce-image')
what = '{name} component image'.format(name=name)
check_subprocesses_to_logfile(what, logfile, [command])
return what
def __build_with_docker(self, repository, build_version):
logging.warning('DOCKER builds are still under development')
name = repository.name
docker_tag = '{reg}/{name}:{build_version}'.format(
reg=self.options.docker_registry,
name=name, build_version=build_version)
cmds = [
'docker build -f Dockerfile -t %s .' % docker_tag,
'docker push %s' % docker_tag
]
gradle_dir = repository.gradle_dir
logfile = self.get_logfile_path(name + '-docker-build')
check_subprocesses_to_logfile(
name + ' docker build', logfile, cmds, cwd=gradle_dir)
def do_run_subprocess_ok(self, check, logfile=None):
if os.path.exists('/bin/true'):
true_path = '/bin/true'
elif os.path.exists('/usr/bin/true'):
true_path = '/usr/bin/true'
else:
raise NotImplementedError('Unsupported test platform.')
tests = [(true_path, ''), ('/bin/echo Hello', 'Hello'),
('/bin/echo "Hello"', 'Hello'),
('/bin/echo "Hello World"', 'Hello World'),
('/bin/echo "Hello\nWorld"', 'Hello\nWorld'),
('/bin/echo \'"Hello World"\'', '"Hello World"')]
for cmd, expect in tests:
if logfile:
output = check_subprocesses_to_logfile('Test Logfile', logfile,
[cmd])
elif check:
output = check_subprocess(cmd)
else:
code, output = run_subprocess(cmd)
self.assertEqual(0, code)
if logfile:
self.assertTrue(os.path.exists(logfile))
self.assertIsNone(output)
with io.open(logfile, 'r', encoding='utf-8') as stream:
lines = stream.read().split('\n')
self.assertTrue('Spawning' in lines[0])
self.assertTrue('process completed with' in lines[-2])
body = '\n'.join(lines[3:-3]).strip()
self.assertEqual(expect, body)
else:
self.assertEqual(expect, output)
def do_run_subprocess_ok(self, check, logfile=None):
if os.path.exists('/bin/true'):
true_path = '/bin/true'
elif os.path.exists('/usr/bin/true'):
true_path = '/usr/bin/true'
else:
raise NotImplementedError('Unsupported test platform.')
tests = [(true_path, ''),
('/bin/echo Hello', 'Hello'),
('/bin/echo "Hello"', 'Hello'),
('/bin/echo "Hello World"', 'Hello World'),
('/bin/echo "Hello\nWorld"', 'Hello\nWorld'),
('/bin/echo \'"Hello World"\'', '"Hello World"')]
for cmd, expect in tests:
if logfile:
output = check_subprocesses_to_logfile('Test Logfile', logfile, [cmd])
elif check:
output = check_subprocess(cmd)
else:
code, output = run_subprocess(cmd)
self.assertEqual(0, code)
if logfile:
self.assertTrue(os.path.exists(logfile))
self.assertIsNone(output)
with io.open(logfile, 'r', encoding='utf-8') as stream:
lines = stream.read().split('\n')
self.assertTrue('Spawning' in lines[0])
self.assertTrue('process completed with' in lines[-2])
body = '\n'.join(lines[3:-3]).strip()
self.assertEqual(expect, body)
else:
self.assertEqual(expect, output)
def _do_repository(self, repository):
"""Implements RepositoryCommandProcessor interface."""
name = repository.name
build_component_image_sh = os.path.join(
os.path.dirname(__file__), '..', 'build_google_component_image.sh')
options = self.options
bom_version = self.source_code_manager.determine_bom_version()
command_line = [
build_component_image_sh,
'--artifact ', name,
'--account', options.build_gce_service_account,
'--hal_daemon_endpoint', 'http://' + options.halyard_daemon,
'--build_project', options.build_gce_project,
'--install_script', options.install_image_script,
'--publish_project', self.__image_project,
'--publish_script', options.publish_gce_image_script,
'--version', bom_version,
'--zone', options.build_gce_zone]
command_line.extend(self.__determine_repo_install_args(repository))
extra_install_args = []
if options.halyard_bom_bucket:
extra_install_args.extend(
['--halyard_config_bucket', options.halyard_bom_bucket])
if options.bintray_debian_repository:
bintray_url = 'https://dl.bintray.com/{org}/{repo}'.format(
org=options.bintray_org,
repo=options.bintray_debian_repository)
extra_install_args.extend([
'--release_track', options.halyard_release_track,
'--halyard_repository', bintray_url,
'--spinnaker_repository', bintray_url])
if extra_install_args:
command_line.extend(['--extra_install_script_args',
'"{0}"'.format(' '.join(extra_install_args))])
command = ' '.join(command_line)
logfile = self.get_logfile_path(name + '-gce-image')
what = u'{name} component image'.format(name=name)
check_subprocesses_to_logfile(what, logfile, [command])
return what
def _promote_halyard(self, repository):
"""Promote an existing build to become the halyard stable version."""
options = self.options
logfile = self.get_logfile_path('promote-all')
env = dict(os.environ)
env.update({
'PUBLISH_HALYARD_BUCKET_BASE_URL': options.halyard_bucket_base_url,
'PUBLISH_HALYARD_DOCKER_IMAGE_BASE': options.halyard_docker_image_base
})
check_subprocesses_to_logfile(
'Promote Halyard', logfile,
['gcloud docker -a', # if repo is private it needs authenticated
'./release/promote-all.sh {candidate} {stable}'.format(
candidate=options.halyard_version,
stable=self.__stable_version),
'./release/promote-all.sh {candidate} stable'.format(
candidate=options.halyard_version)],
env=env,
cwd=repository.git_dir)
def _promote_halyard(self, repository):
"""Promote an existing build to become the halyard stable version."""
options = self.options
logfile = self.get_logfile_path('promote-all')
env = dict(os.environ)
env.update({
'PUBLISH_HALYARD_BUCKET_BASE_URL': options.halyard_bucket_base_url,
'PUBLISH_HALYARD_DOCKER_IMAGE_BASE': options.halyard_docker_image_base
})
check_subprocesses_to_logfile(
'Promote Halyard', logfile,
['gcloud docker -a', # if repo is private it needs authenticated
'./release/promote-all.sh {candidate} {stable}'.format(
candidate=options.halyard_version,
stable=self.__stable_version),
'./release/promote-all.sh {candidate} stable'.format(
candidate=options.halyard_version)],
env=env,
cwd=repository.git_dir)
def __build_with_docker(self, repository, build_version):
logging.warning('DOCKER builds are still under development')
name = repository.name
docker_tag = '{reg}/{name}:{build_version}'.format(
reg=self.options.docker_registry,
name=name,
build_version=build_version)
cmds = [
'docker build -f Dockerfile -t %s .' % docker_tag,
'docker push %s' % docker_tag
]
gradle_dir = repository.gradle_dir
logfile = self.get_logfile_path(name + '-docker-build')
check_subprocesses_to_logfile(name + ' docker build',
logfile,
cmds,
cwd=gradle_dir)
def publish_to_nightly(self, repository):
options = self.options
cmd = './release/promote-all.sh {version} nightly'.format(
version=self.__build_version)
env = dict(os.environ)
env.update({
'PUBLISH_HALYARD_BUCKET_BASE_URL': options.halyard_bucket_base_url,
'PUBLISH_HALYARD_DOCKER_IMAGE_BASE': options.halyard_docker_image_base
})
logging.info(
'Preparing the environment variables for release/all.sh:\n'
' PUBLISH_HALYARD_DOCKER_IMAGE_BASE=%s\n'
' PUBLISH_HALYARD_BUCKET_BASE_URL=%s',
options.halyard_docker_image_base,
options.halyard_bucket_base_url)
logfile = self.get_logfile_path('halyard-publish-to-nightly')
check_subprocesses_to_logfile(
'halyard publish to nightly', logfile,
[cmd], cwd=repository.git_dir, env=env)
def _do_repository(self, repository):
generate_path = self.options.generate_swagger_path
if not generate_path:
self.__build_from_live_server(repository)
return
swagger_dir = os.path.dirname(generate_path)
generate_logfile = self.get_logfile_path(
repository.name + '-generate_swagger')
check_subprocesses_to_logfile(
'Extracting API to JSON', generate_logfile, [generate_path],
cwd=repository.git_dir)
docs_dir = self.get_output_dir()
ensure_dir_exists(docs_dir)
json_path = os.path.join(docs_dir, 'docs.json')
swagger_output_path = os.path.join(
repository.git_dir, swagger_dir, 'swagger.json')
with open(swagger_output_path, 'r') as stream:
content = stream.read()
with open(json_path, 'w') as stream:
stream.write(content)
self.build_swagger_docs(repository, json_path)
def _do_repository(self, repository):
generate_path = self.options.generate_swagger_path
if not generate_path:
self.__build_from_live_server(repository)
return
swagger_dir = os.path.dirname(generate_path)
generate_logfile = self.get_logfile_path(repository.name +
'-generate_swagger')
check_subprocesses_to_logfile('Extracting API to JSON',
generate_logfile, [generate_path],
cwd=repository.git_dir)
docs_dir = self.get_output_dir()
ensure_dir_exists(docs_dir)
json_path = os.path.join(docs_dir, 'docs.json')
swagger_output_path = os.path.join(repository.git_dir, swagger_dir,
'swagger.json')
with open(swagger_output_path, 'r') as stream:
content = stream.read()
with open(json_path, 'w') as stream:
stream.write(content)
self.build_swagger_docs(repository, json_path)
def build_all_halyard_deployments(self, repository):
"""Helper function for building halyard."""
options = self.options
git_dir = repository.git_dir
source_info = self.source_code_manager.lookup_source_info(repository)
self.__build_version = source_info.to_build_version()
cmd = './release/all.sh {version} nightly'.format(
version=self.__build_version)
env = dict(os.environ)
logging.info(
'Preparing the environment variables for release/all.sh:\n'
' PUBLISH_HALYARD_DOCKER_IMAGE_BASE=%s\n'
' PUBLISH_HALYARD_BUCKET_BASE_URL=%s',
options.halyard_docker_image_base, options.halyard_bucket_base_url)
env['PUBLISH_HALYARD_DOCKER_IMAGE_BASE'] = options.halyard_docker_image_base
env['PUBLISH_HALYARD_BUCKET_BASE_URL'] = options.halyard_bucket_base_url
logfile = self.get_logfile_path('jar-build')
check_subprocesses_to_logfile('{name} build'.format(name='halyard'),
logfile, [cmd],
cwd=git_dir,
env=env)
def do_run_subprocess_ok(self, check, logfile=None):
if os.path.exists("/bin/true"):
true_path = "/bin/true"
elif os.path.exists("/usr/bin/true"):
true_path = "/usr/bin/true"
else:
raise NotImplementedError("Unsupported test platform.")
tests = [
(true_path, ""),
("/bin/echo Hello", "Hello"),
('/bin/echo "Hello"', "Hello"),
('/bin/echo "Hello World"', "Hello World"),
('/bin/echo "Hello\nWorld"', "Hello\nWorld"),
("/bin/echo '\"Hello World\"'", '"Hello World"'),
]
for cmd, expect in tests:
if logfile:
output = check_subprocesses_to_logfile("Test Logfile", logfile,
[cmd])
elif check:
output = check_subprocess(cmd)
else:
code, output = run_subprocess(cmd)
self.assertEqual(0, code)
if logfile:
self.assertTrue(os.path.exists(logfile))
self.assertIsNone(output)
with io.open(logfile, "r", encoding="utf-8") as stream:
lines = stream.read().split("\n")
self.assertTrue("Spawning" in lines[0])
self.assertTrue("process completed with" in lines[-2])
body = "\n".join(lines[3:-3]).strip()
self.assertEqual(expect, body)
else:
self.assertEqual(expect, output)
def _do_repository(self, repository):
"""Implements RepositoryCommandProcessor interface."""
name = repository.name
build_component_image_sh = os.path.join(
os.path.dirname(__file__), "..", "build_google_component_image.sh")
options = self.options
bom_version = self.source_code_manager.determine_bom_version()
command_line = [
build_component_image_sh,
"--artifact ",
name,
"--account",
options.build_gce_service_account,
"--hal_daemon_endpoint",
"http://" + options.halyard_daemon,
"--build_project",
options.build_gce_project,
"--install_script",
options.install_image_script,
"--publish_project",
self.__image_project,
"--publish_script",
options.publish_gce_image_script,
"--version",
bom_version,
"--zone",
options.build_gce_zone,
]
command_line.extend(self.__determine_repo_install_args(repository))
extra_install_args = []
if options.halyard_bom_bucket:
extra_install_args.extend(
["--halyard_config_bucket", options.halyard_bom_bucket])
if options.bintray_debian_repository:
bintray_url = "https://dl.bintray.com/{org}/{repo}".format(
org=options.bintray_org,
repo=options.bintray_debian_repository)
extra_install_args.extend([
"--release_track",
options.halyard_release_track,
"--halyard_repository",
bintray_url,
"--spinnaker_repository",
bintray_url,
])
if extra_install_args:
command_line.extend([
"--extra_install_script_args",
'"{0}"'.format(" ".join(extra_install_args)),
])
command = " ".join(command_line)
logfile = self.get_logfile_path(name + "-gce-image")
what = "{name} component image".format(name=name)
check_subprocesses_to_logfile(what, logfile, [command])
return what
