def build_all_halyard_deployments(self, repository): """Helper function for building halyard.""" options = self.options git_dir = repository.git_dir summary = self.source_code_manager.git.collect_repository_summary(git_dir) self.__build_version = '%s-%s' % (summary.version, options.build_number) cmd = './release/all.sh {version} nightly'.format( version=self.__build_version) env = dict(os.environ) env.update({ 'PUBLISH_HALYARD_BUCKET_BASE_URL': options.halyard_bucket_base_url, 'PUBLISH_HALYARD_DOCKER_IMAGE_BASE': options.halyard_docker_image_base }) logging.info( 'Preparing the environment variables for release/all.sh:\n' ' PUBLISH_HALYARD_DOCKER_IMAGE_BASE=%s\n' ' PUBLISH_HALYARD_BUCKET_BASE_URL=%s', options.halyard_docker_image_base, options.halyard_bucket_base_url) logfile = self.get_logfile_path('jar-build') check_subprocesses_to_logfile( '{name} build'.format(name='halyard'), logfile, [cmd], cwd=git_dir, env=env)
def build_halyard_docs(command, repository): """Builds Halyard's CLI and updates documentation in its repo.""" cli_dir = os.path.join(repository.git_dir, 'halyard-cli') # Before, we were doing this first: # check_run_quick('git -C halyard rev-parse HEAD' # ' | xargs git -C halyard checkout ;') # however now the repository should already be at the desired commit. logging.debug('Building Halyard CLI and docs.') logfile = command.get_logfile_path('build-docs') check_subprocesses_to_logfile( 'Build halyard docs', logfile, ['make'], cwd=cli_dir)
def test_check_to_file_subprocess_failed(self): cmd = '/bin/ls /abc/def' path = os.path.join(self.base_temp_dir, 'check_failed.log') with self.assertRaises(ExecutionError) as ex: check_subprocesses_to_logfile('Test Logfile', path, [cmd]) self.assertTrue(hasattr(ex.exception, 'loggedit')) self.assertTrue(os.path.exists(path)) with open(path, 'r') as stream: lines = stream.read().split('\n') body = '\n'.join(lines[3:-3]).strip() expect = "/bin/ls: cannot access '/abc/def': No such file or directory" self.assertEqual(expect, body)
def test_check_to_file_subprocess_failed(self): cmd = "/bin/ls /abc/def" path = os.path.join(self.base_temp_dir, "check_failed.log") with self.assertRaises(ExecutionError) as ex: check_subprocesses_to_logfile("Test Logfile", path, [cmd]) self.assertTrue(hasattr(ex.exception, "loggedit")) self.assertTrue(os.path.exists(path)) with open(path, "r") as stream: lines = stream.read().split("\n") body = "\n".join(lines[3:-3]).strip() expect = "/bin/ls: cannot access '/abc/def': No such file or directory" self.assertEqual(expect, body)
def run_and_log_test_script(command): logfile = os.path.join(self.options.output_dir, 'citest_logs', '%s-%s.console.log' % (test_name, os.getpid())) logging.info('Logging test "%s" to %s', test_name, logfile) try: check_subprocesses_to_logfile('running test', logfile, [command]) retcode = 0 logging.info('Test %s PASSED -- see %s', test_name, logfile) except: retcode = -1 logging.info('Test %s FAILED -- see %s', test_name, logfile) return retcode, logfile
def _build_release(self, repository): """Rebuild the actual release debian package. We dont necessarily need to rebuild here. We just need to push as debian to the "-stable". However there isnt an easy way to do this. Note that this is not the promoted version. For safety[*] and simplicity we'll promote the candidate whose version was used to build this. Ideally this function can go away. [*] Safety because the candidate was tested whereas this build was not. """ # Ideally we would just modify the existing bintray version to add # *-stable to the distributions, however it does not appear possible # to patch the debian attributes of a bintray version, only the # version metadata. Therefore, we'll rebuild it. # Alternatively we could download the existing and push a new one, # however I dont see how to get at the existing debian metadata and # dont want to ommit something options = self.options git_dir = repository.git_dir summary = self.__scm.git.collect_repository_summary(git_dir) config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cloudbuild", "debs.yml") substitutions = { "_BRANCH_NAME": options.git_branch, "_BRANCH_TAG": re.sub(r"\W", "_", options.git_branch), "_BUILD_NUMBER": options.build_number, "_IMAGE_NAME": "halyard", "_VERSION": summary.version, } # Convert it to the format expected by gcloud: "_FOO=bar,_BAZ=qux" substitutions_arg = ",".join("=".join((str(k), str(v))) for k, v in substitutions.items()) command = ("gcloud builds submit " " --account={account} --project={project}" " --substitutions={substitutions_arg}," " --config={config} .".format( account=options.gcb_service_account, project=options.gcb_project, substitutions_arg=substitutions_arg, config=config_path, )) logfile = self.get_logfile_path("build-deb") check_subprocesses_to_logfile("building deb with published version", logfile, [command], cwd=git_dir)
def _do_repository(self, repository): """Implements RepositoryCommandProcessor interface.""" name = repository.name build_component_image_sh = os.path.join( os.path.dirname(__file__), '..', 'build_google_component_image.sh') options = self.options bom_version = self.source_code_manager.determine_bom_version() command_line = [ build_component_image_sh, '--artifact ', name, '--account', options.build_gce_service_account, '--hal_daemon_endpoint', 'http://' + options.halyard_daemon, '--build_project', options.build_gce_project, '--install_script', options.install_image_script, '--publish_project', self.__image_project, '--publish_script', options.publish_gce_image_script, '--version', bom_version, '--zone', options.build_gce_zone ] command_line.extend(self.__determine_repo_install_args(repository)) extra_install_args = [] if options.halyard_bom_bucket: extra_install_args.extend( ['--halyard_config_bucket', options.halyard_bom_bucket]) if options.bintray_debian_repository: bintray_url = 'https://dl.bintray.com/{org}/{repo}'.format( org=options.bintray_org, repo=options.bintray_debian_repository) extra_install_args.extend([ '--release_track', options.halyard_release_track, '--halyard_repository', bintray_url, '--spinnaker_repository', bintray_url ]) if extra_install_args: command_line.extend([ '--extra_install_script_args', '"{0}"'.format(' '.join(extra_install_args)) ]) command = ' '.join(command_line) logfile = self.get_logfile_path(name + '-gce-image') what = '{name} component image'.format(name=name) check_subprocesses_to_logfile(what, logfile, [command]) return what
def __build_with_docker(self, repository, build_version): logging.warning('DOCKER builds are still under development') name = repository.name docker_tag = '{reg}/{name}:{build_version}'.format( reg=self.options.docker_registry, name=name, build_version=build_version) cmds = [ 'docker build -f Dockerfile -t %s .' % docker_tag, 'docker push %s' % docker_tag ] gradle_dir = repository.gradle_dir logfile = self.get_logfile_path(name + '-docker-build') check_subprocesses_to_logfile( name + ' docker build', logfile, cmds, cwd=gradle_dir)
def do_run_subprocess_ok(self, check, logfile=None): if os.path.exists('/bin/true'): true_path = '/bin/true' elif os.path.exists('/usr/bin/true'): true_path = '/usr/bin/true' else: raise NotImplementedError('Unsupported test platform.') tests = [(true_path, ''), ('/bin/echo Hello', 'Hello'), ('/bin/echo "Hello"', 'Hello'), ('/bin/echo "Hello World"', 'Hello World'), ('/bin/echo "Hello\nWorld"', 'Hello\nWorld'), ('/bin/echo \'"Hello World"\'', '"Hello World"')] for cmd, expect in tests: if logfile: output = check_subprocesses_to_logfile('Test Logfile', logfile, [cmd]) elif check: output = check_subprocess(cmd) else: code, output = run_subprocess(cmd) self.assertEqual(0, code) if logfile: self.assertTrue(os.path.exists(logfile)) self.assertIsNone(output) with io.open(logfile, 'r', encoding='utf-8') as stream: lines = stream.read().split('\n') self.assertTrue('Spawning' in lines[0]) self.assertTrue('process completed with' in lines[-2]) body = '\n'.join(lines[3:-3]).strip() self.assertEqual(expect, body) else: self.assertEqual(expect, output)
def _do_repository(self, repository): """Implements RepositoryCommandProcessor interface.""" name = repository.name build_component_image_sh = os.path.join( os.path.dirname(__file__), '..', 'build_google_component_image.sh') options = self.options bom_version = self.source_code_manager.determine_bom_version() command_line = [ build_component_image_sh, '--artifact ', name, '--account', options.build_gce_service_account, '--hal_daemon_endpoint', 'http://' + options.halyard_daemon, '--build_project', options.build_gce_project, '--install_script', options.install_image_script, '--publish_project', self.__image_project, '--publish_script', options.publish_gce_image_script, '--version', bom_version, '--zone', options.build_gce_zone] command_line.extend(self.__determine_repo_install_args(repository)) extra_install_args = [] if options.halyard_bom_bucket: extra_install_args.extend( ['--halyard_config_bucket', options.halyard_bom_bucket]) if options.bintray_debian_repository: bintray_url = 'https://dl.bintray.com/{org}/{repo}'.format( org=options.bintray_org, repo=options.bintray_debian_repository) extra_install_args.extend([ '--release_track', options.halyard_release_track, '--halyard_repository', bintray_url, '--spinnaker_repository', bintray_url]) if extra_install_args: command_line.extend(['--extra_install_script_args', '"{0}"'.format(' '.join(extra_install_args))]) command = ' '.join(command_line) logfile = self.get_logfile_path(name + '-gce-image') what = u'{name} component image'.format(name=name) check_subprocesses_to_logfile(what, logfile, [command]) return what
def _promote_halyard(self, repository): """Promote an existing build to become the halyard stable version.""" options = self.options logfile = self.get_logfile_path('promote-all') env = dict(os.environ) env.update({ 'PUBLISH_HALYARD_BUCKET_BASE_URL': options.halyard_bucket_base_url, 'PUBLISH_HALYARD_DOCKER_IMAGE_BASE': options.halyard_docker_image_base }) check_subprocesses_to_logfile( 'Promote Halyard', logfile, ['gcloud docker -a', # if repo is private it needs authenticated './release/promote-all.sh {candidate} {stable}'.format( candidate=options.halyard_version, stable=self.__stable_version), './release/promote-all.sh {candidate} stable'.format( candidate=options.halyard_version)], env=env, cwd=repository.git_dir)
def __build_with_docker(self, repository, build_version): logging.warning('DOCKER builds are still under development') name = repository.name docker_tag = '{reg}/{name}:{build_version}'.format( reg=self.options.docker_registry, name=name, build_version=build_version) cmds = [ 'docker build -f Dockerfile -t %s .' % docker_tag, 'docker push %s' % docker_tag ] gradle_dir = repository.gradle_dir logfile = self.get_logfile_path(name + '-docker-build') check_subprocesses_to_logfile(name + ' docker build', logfile, cmds, cwd=gradle_dir)
def publish_to_nightly(self, repository): options = self.options cmd = './release/promote-all.sh {version} nightly'.format( version=self.__build_version) env = dict(os.environ) env.update({ 'PUBLISH_HALYARD_BUCKET_BASE_URL': options.halyard_bucket_base_url, 'PUBLISH_HALYARD_DOCKER_IMAGE_BASE': options.halyard_docker_image_base }) logging.info( 'Preparing the environment variables for release/all.sh:\n' ' PUBLISH_HALYARD_DOCKER_IMAGE_BASE=%s\n' ' PUBLISH_HALYARD_BUCKET_BASE_URL=%s', options.halyard_docker_image_base, options.halyard_bucket_base_url) logfile = self.get_logfile_path('halyard-publish-to-nightly') check_subprocesses_to_logfile( 'halyard publish to nightly', logfile, [cmd], cwd=repository.git_dir, env=env)
def _do_repository(self, repository): generate_path = self.options.generate_swagger_path if not generate_path: self.__build_from_live_server(repository) return swagger_dir = os.path.dirname(generate_path) generate_logfile = self.get_logfile_path( repository.name + '-generate_swagger') check_subprocesses_to_logfile( 'Extracting API to JSON', generate_logfile, [generate_path], cwd=repository.git_dir) docs_dir = self.get_output_dir() ensure_dir_exists(docs_dir) json_path = os.path.join(docs_dir, 'docs.json') swagger_output_path = os.path.join( repository.git_dir, swagger_dir, 'swagger.json') with open(swagger_output_path, 'r') as stream: content = stream.read() with open(json_path, 'w') as stream: stream.write(content) self.build_swagger_docs(repository, json_path)
def _do_repository(self, repository): generate_path = self.options.generate_swagger_path if not generate_path: self.__build_from_live_server(repository) return swagger_dir = os.path.dirname(generate_path) generate_logfile = self.get_logfile_path(repository.name + '-generate_swagger') check_subprocesses_to_logfile('Extracting API to JSON', generate_logfile, [generate_path], cwd=repository.git_dir) docs_dir = self.get_output_dir() ensure_dir_exists(docs_dir) json_path = os.path.join(docs_dir, 'docs.json') swagger_output_path = os.path.join(repository.git_dir, swagger_dir, 'swagger.json') with open(swagger_output_path, 'r') as stream: content = stream.read() with open(json_path, 'w') as stream: stream.write(content) self.build_swagger_docs(repository, json_path)
def build_all_halyard_deployments(self, repository): """Helper function for building halyard.""" options = self.options git_dir = repository.git_dir source_info = self.source_code_manager.lookup_source_info(repository) self.__build_version = source_info.to_build_version() cmd = './release/all.sh {version} nightly'.format( version=self.__build_version) env = dict(os.environ) logging.info( 'Preparing the environment variables for release/all.sh:\n' ' PUBLISH_HALYARD_DOCKER_IMAGE_BASE=%s\n' ' PUBLISH_HALYARD_BUCKET_BASE_URL=%s', options.halyard_docker_image_base, options.halyard_bucket_base_url) env['PUBLISH_HALYARD_DOCKER_IMAGE_BASE'] = options.halyard_docker_image_base env['PUBLISH_HALYARD_BUCKET_BASE_URL'] = options.halyard_bucket_base_url logfile = self.get_logfile_path('jar-build') check_subprocesses_to_logfile('{name} build'.format(name='halyard'), logfile, [cmd], cwd=git_dir, env=env)
def do_run_subprocess_ok(self, check, logfile=None): if os.path.exists("/bin/true"): true_path = "/bin/true" elif os.path.exists("/usr/bin/true"): true_path = "/usr/bin/true" else: raise NotImplementedError("Unsupported test platform.") tests = [ (true_path, ""), ("/bin/echo Hello", "Hello"), ('/bin/echo "Hello"', "Hello"), ('/bin/echo "Hello World"', "Hello World"), ('/bin/echo "Hello\nWorld"', "Hello\nWorld"), ("/bin/echo '\"Hello World\"'", '"Hello World"'), ] for cmd, expect in tests: if logfile: output = check_subprocesses_to_logfile("Test Logfile", logfile, [cmd]) elif check: output = check_subprocess(cmd) else: code, output = run_subprocess(cmd) self.assertEqual(0, code) if logfile: self.assertTrue(os.path.exists(logfile)) self.assertIsNone(output) with io.open(logfile, "r", encoding="utf-8") as stream: lines = stream.read().split("\n") self.assertTrue("Spawning" in lines[0]) self.assertTrue("process completed with" in lines[-2]) body = "\n".join(lines[3:-3]).strip() self.assertEqual(expect, body) else: self.assertEqual(expect, output)
def _do_repository(self, repository): """Implements RepositoryCommandProcessor interface.""" name = repository.name build_component_image_sh = os.path.join( os.path.dirname(__file__), "..", "build_google_component_image.sh") options = self.options bom_version = self.source_code_manager.determine_bom_version() command_line = [ build_component_image_sh, "--artifact ", name, "--account", options.build_gce_service_account, "--hal_daemon_endpoint", "http://" + options.halyard_daemon, "--build_project", options.build_gce_project, "--install_script", options.install_image_script, "--publish_project", self.__image_project, "--publish_script", options.publish_gce_image_script, "--version", bom_version, "--zone", options.build_gce_zone, ] command_line.extend(self.__determine_repo_install_args(repository)) extra_install_args = [] if options.halyard_bom_bucket: extra_install_args.extend( ["--halyard_config_bucket", options.halyard_bom_bucket]) if options.bintray_debian_repository: bintray_url = "https://dl.bintray.com/{org}/{repo}".format( org=options.bintray_org, repo=options.bintray_debian_repository) extra_install_args.extend([ "--release_track", options.halyard_release_track, "--halyard_repository", bintray_url, "--spinnaker_repository", bintray_url, ]) if extra_install_args: command_line.extend([ "--extra_install_script_args", '"{0}"'.format(" ".join(extra_install_args)), ]) command = " ".join(command_line) logfile = self.get_logfile_path(name + "-gce-image") what = "{name} component image".format(name=name) check_subprocesses_to_logfile(what, logfile, [command]) return what