def _do_command(self): options = self.options gist_url = options.build_changelog_gist_url index = gist_url.rfind('/') if index < 0: index = gist_url.rfind(':') # ssh gist gist_id = gist_url[index + 1:] git_dir = os.path.join(self.get_input_dir(), gist_id) if not os.path.exists(git_dir): logging.debug('Cloning gist from %s', gist_url) ensure_dir_exists(os.path.dirname(git_dir)) self.__git.check_run(os.path.dirname(git_dir), 'clone ' + gist_url) else: logging.debug('Updating gist in "%s"', git_dir) self.__git.check_run(git_dir, 'fetch origin master') self.__git.check_run(git_dir, 'checkout master') dest_path = os.path.join( git_dir, '%s-raw-changelog.md' % options.git_branch) logging.debug('Copying "%s" to "%s"', options.changelog_path, dest_path) shutil.copyfile(options.changelog_path, dest_path) self.__git.check_run(git_dir, 'add ' + os.path.basename(dest_path)) self.__git.check_commit_or_no_changes( git_dir, '-a -m "Updated %s"' % os.path.basename(dest_path)) logging.debug('Pushing back gist') self.__git.check_run(git_dir, 'push -f origin master')
def _do_command(self): options = self.options gist_url = options.build_changelog_gist_url index = gist_url.rfind('/') if index < 0: index = gist_url.rfind(':') # ssh gist gist_id = gist_url[index + 1:] git_dir = os.path.join(self.get_input_dir(), gist_id) if not os.path.exists(git_dir): logging.debug('Cloning gist from %s', gist_url) ensure_dir_exists(os.path.dirname(git_dir)) self.git_run_with_retries(os.path.dirname(git_dir), 'clone ' + gist_url) else: logging.debug('Updating gist in "%s"', git_dir) self.git_run_with_retries(git_dir, 'fetch origin master') self.git_run_with_retries(git_dir, 'checkout master') dest_path = os.path.join(git_dir, '%s-raw-changelog.md' % options.git_branch) logging.debug('Copying "%s" to "%s"', options.changelog_path, dest_path) shutil.copyfile(options.changelog_path, dest_path) self.git_run_with_retries(git_dir, 'add ' + os.path.basename(dest_path)) self.__git.check_commit_or_no_changes( git_dir, '-a -m "Updated %s"' % os.path.basename(dest_path)) logging.debug('Pushing back gist') self.git_run_with_retries(git_dir, 'push -f origin master')
def __generate_json_from_url( self, repository, server_url, output_path): """Build the swagger json from the swagger endpoint.""" ensure_dir_exists(os.path.dirname(output_path)) logging.info('Generating swagger docs for %s', repository.name) check_subprocess('curl -s {url} -o {output_path}' .format(url=server_url, output_path=output_path))
def _do_repository(self, repository): generate_path = self.options.generate_swagger_path if not generate_path: self.__build_from_live_server(repository) return swagger_dir = os.path.dirname(generate_path) generate_logfile = self.get_logfile_path(repository.name + "-generate_swagger") check_subprocesses_to_logfile( "Extracting API to JSON", generate_logfile, [generate_path], cwd=repository.git_dir, ) docs_dir = self.get_output_dir() ensure_dir_exists(docs_dir) json_path = os.path.join(docs_dir, "docs.json") swagger_output_path = os.path.join( repository.git_dir, swagger_dir, "swagger.json" ) with open(swagger_output_path, "r") as stream: content = stream.read() with open(json_path, "w") as stream: stream.write(content) self.build_swagger_docs(repository, json_path)
def clone_repository_to_path( self, repository, commit=None, branch=None, default_branch=None): """Clone the remote repository at the given commit or branch. If requesting a branch and it is not found, then settle for the default branch, if one was explicitly specified. """ # pylint: disable=too-many-arguments if (commit != None) and (branch != None): raise_and_log_error( ConfigError('At most one of commit or branch can be specified.')) origin = repository.origin parts = self.normalize_repo_url(repository.origin) if len(parts) == 3: pull_url = (self.make_ssh_url(*parts) if self.__options.github_pull_ssh else self.make_https_url(*parts)) else: pull_url = origin git_dir = repository.git_dir logging.debug('Begin cloning %s', pull_url) parent_dir = os.path.dirname(git_dir) ensure_dir_exists(parent_dir) clone_command = 'clone ' + pull_url if branch: branches = [branch] if default_branch: branches.append(default_branch) self.__check_clone_branch(pull_url, parent_dir, clone_command, branches) else: self.check_run(parent_dir, clone_command) logging.info('Cloned %s into %s', pull_url, parent_dir) if commit: self.check_run(git_dir, 'checkout -q ' + commit, echo=True) upstream = repository.upstream_or_none() if upstream and not self.is_same_repo(upstream, origin): logging.debug('Adding upstream %s with disabled push', upstream) self.check_run(git_dir, 'remote add upstream ' + upstream) which = ('upstream' if upstream and not self.is_same_repo(upstream, origin) else 'origin') if self.__options.github_disable_upstream_push: self.check_run( git_dir, 'remote set-url --push {which} disabled'.format(which=which)) if which != 'origin' or not self.__options.github_disable_upstream_push: parts = self.normalize_repo_url(repository.origin) if len(parts) == 3: # Origin is not a local path logging.debug('Fixing origin push url') push_url = (self.make_ssh_url(*parts) if self.__options.github_push_ssh else self.make_https_url(*parts)) self.check_run(git_dir, 'remote set-url --push origin ' + push_url) logging.debug('Finished cloning %s', pull_url)
def check_subprocesses_to_logfile(what, logfile, cmds, append=False, **kwargs): """Wrapper around check_subprocess that logs output to a logfile. Args: what: [string] For logging purposes, what is the command for. logfile: [path] The logfile to write to. cmds: [list of string] A list of commands to run. append: [boolean] Open the log file as append if true, write new default. kwargs: [kwargs] Additional keyword arguments to pass to check_subprocess. """ mode = 'a' if append else 'w' how = 'Appending' if append else 'Logging' logging.info('%s %s to %s', how, what, logfile) ensure_dir_exists(os.path.dirname(logfile)) with open(logfile, mode) as stream: try: check_subprocess_sequence(cmds, stream=stream, embed_errors=False, **kwargs) except Exception: logging.error('%s failed. Log file [%s] follows:', what, logfile) with open(logfile, 'r') as readagain: output = readagain.read() log_embedded_output(logging.ERROR, logfile, output) logging.error('%s failed. See embedded logfile above', what) ensure_dir_exists(ERROR_LOGFILE_DIR) error_path = os.path.join('errors', os.path.basename(logfile)) logging.info('Copying error log file to %s', error_path) with open(error_path, 'w') as f: f.write(output) raise
def clone_repository_to_path( self, repository, commit=None, branch=None, default_branch=None ): """Clone the remote repository at the given commit or branch. If requesting a branch and it is not found, then settle for the default branch, if one was explicitly specified. """ # pylint: disable=too-many-arguments if (commit is not None) and (branch is not None): raise_and_log_error( ConfigError("At most one of commit or branch can be specified.") ) pull_url = self.determine_pull_url(repository.origin) git_dir = repository.git_dir logging.debug("Begin cloning %s", pull_url) parent_dir = os.path.dirname(git_dir) ensure_dir_exists(parent_dir) clone_command = "clone " + pull_url if branch: branches = [branch] if default_branch: branches.append(default_branch) self.__check_clone_branch(pull_url, parent_dir, clone_command, branches) else: self.check_run(parent_dir, clone_command) logging.info("Cloned %s into %s", pull_url, parent_dir) if commit: self.checkout(repository, commit) upstream = repository.upstream_or_none() origin = repository.origin if upstream and not self.is_same_repo(upstream, origin): logging.debug("Adding upstream %s with disabled push", upstream) self.check_run(git_dir, "remote add upstream " + upstream) which = ( "upstream" if upstream and not self.is_same_repo(upstream, origin) else "origin" ) if self.__options.github_disable_upstream_push: self.check_run( git_dir, "remote set-url --push {which} disabled".format(which=which) ) if which != "origin" or not self.__options.github_disable_upstream_push: parts = self.normalize_repo_url(repository.origin) if len(parts) == 3: # Origin is not a local path logging.debug("Fixing origin push url") push_url = self.determine_push_url(repository.origin) self.check_run(git_dir, "remote set-url --push origin " + push_url) logging.debug("Finished cloning %s", pull_url)
def test_ensure_dir(self): want = os.path.join(self.base_temp_dir, "ensure", "a", "b", "c") self.assertFalse(os.path.exists(want)) ensure_dir_exists(want) self.assertTrue(os.path.exists(want)) # Ok if already exists ensure_dir_exists(want) self.assertTrue(os.path.exists(want))
def __generate_json_from_url(self, repository, server_url, output_path): """Build the swagger json from the swagger endpoint.""" ensure_dir_exists(os.path.dirname(output_path)) logging.info("Generating swagger docs for %s", repository.name) check_subprocess( "curl -s {url} -o {output_path}".format( url=server_url, output_path=output_path ) )
def test_ensure_dir(self): want = os.path.join(self.base_temp_dir, 'ensure', 'a', 'b', 'c') self.assertFalse(os.path.exists(want)) ensure_dir_exists(want) self.assertTrue(os.path.exists(want)) # Ok if already exists ensure_dir_exists(want) self.assertTrue(os.path.exists(want))
def get_output_dir(self, command=None): """Return the output dir for persistent build output from this command.""" command = command or self.__options.command output_command_path = os.path.join(self.__options.output_dir, command) # FIXME: We manually ensure the output dir is there if it doesn't exist. # This should be created before the command is run. if not os.path.isdir(output_command_path): logging.debug('making dir %s', output_command_path) ensure_dir_exists(output_command_path) return output_command_path
def clone_repository_to_path( self, repository, commit=None, branch=None, default_branch=None): """Clone the remote repository at the given commit or branch. If requesting a branch and it is not found, then settle for the default branch, if one was explicitly specified. """ # pylint: disable=too-many-arguments if (commit != None) and (branch != None): raise_and_log_error( ConfigError('At most one of commit or branch can be specified.')) pull_url = self.determine_pull_url(repository) git_dir = repository.git_dir logging.debug('Begin cloning %s', pull_url) parent_dir = os.path.dirname(git_dir) ensure_dir_exists(parent_dir) clone_command = 'clone ' + pull_url if branch: branches = [branch] if default_branch: branches.append(default_branch) self.__check_clone_branch(pull_url, parent_dir, clone_command, branches) else: self.check_run(parent_dir, clone_command) logging.info('Cloned %s into %s', pull_url, parent_dir) if commit: self.check_run(git_dir, 'checkout -q ' + commit, echo=True) upstream = repository.upstream_or_none() origin = repository.origin if upstream and not self.is_same_repo(upstream, origin): logging.debug('Adding upstream %s with disabled push', upstream) self.check_run(git_dir, 'remote add upstream ' + upstream) which = ('upstream' if upstream and not self.is_same_repo(upstream, origin) else 'origin') if self.__options.github_disable_upstream_push: self.check_run( git_dir, 'remote set-url --push {which} disabled'.format(which=which)) if which != 'origin' or not self.__options.github_disable_upstream_push: parts = self.normalize_repo_url(repository.origin) if len(parts) == 3: # Origin is not a local path logging.debug('Fixing origin push url') push_url = self.determine_push_url(repository) self.check_run(git_dir, 'remote set-url --push origin ' + push_url) logging.debug('Finished cloning %s', pull_url)
def __build_from_live_server(self, repository): """Implements CommandProcessor interface.""" docs_url_path = SWAGGER_URL_PATHS[repository.name] env = dict(os.environ) port = unused_port() env['SERVER_PORT'] = str(port) base_url = 'http://localhost:' + str(port) gate_logfile = self.get_logfile_path(repository.name + '-apidocs-server') logging.info( 'Starting up prototype %s so we can extract docs from it.' ' We will log this instance to %s', repository.name, gate_logfile) boot_run_cmd = './gradlew' # default will run ensure_dir_exists(os.path.dirname(gate_logfile)) gate_logstream = open(gate_logfile, 'w') process = start_subprocess(boot_run_cmd, stream=gate_logstream, stdout=gate_logstream, cwd=repository.git_dir, env=env) max_wait_secs = self.options.max_wait_secs_startup # pylint: disable=broad-except try: logging.info('Waiting up to %s secs for %s to be ready on port %d', max_wait_secs, repository.name, port) self.wait_for_url(base_url + '/health', max_wait_secs) json_path = os.path.join(self.get_output_dir(), 'docs.json') self.__generate_json_from_url(repository, base_url + '/' + docs_url_path, json_path) self.build_swagger_docs(repository, json_path) finally: try: gate_logstream.flush() gate_logstream.write( '\n' + log_timestring() + ' ***** buildtool is killing subprocess *****\n') logging.info( 'Killing %s subprocess %s now that we are done with it', repository.name, process.pid) process.kill() wait_subprocess(process) gate_logstream.close() except Exception as ex: maybe_log_exception( self.name, ex, 'Ignoring exception while stopping {name} subprocess {pid}.' .format(name=repository.name, pid=process.pid))
def __collect_halconfig_files(self, repository): """Gets the component config files and writes them into the output_dir.""" name = repository.name if (name not in SPINNAKER_BOM_REPOSITORY_NAMES or name in ['spinnaker']): logging.debug('%s does not use config files -- skipping', name) return if name == 'spinnaker-monitoring': config_root = os.path.join(repository.git_dir, 'spinnaker-monitoring-daemon') else: config_root = repository.git_dir service_name = self.scm.repository_name_to_service_name( repository.name) target_dir = os.path.join(self.get_output_dir(), 'halconfig', service_name) ensure_dir_exists(target_dir) config_path = os.path.join(config_root, 'halconfig') logging.info('Copying configs from %s...', config_path) if not os.path.exists(config_path) and repository.name == 'kayenta': logging.warning('Kayenta does not yet have a halconfig.') return for profile in os.listdir(config_path): profile_path = os.path.join(config_path, profile) if os.path.isfile(profile_path): shutil.copyfile(profile_path, os.path.join(target_dir, profile)) logging.debug('Copied profile to %s', profile_path) elif not os.path.isdir(profile_path): logging.warning('%s is neither file nor directory -- ignoring', profile_path) continue else: tar_path = os.path.join( target_dir, '{profile}.tar.gz'.format(profile=profile)) file_list = ' '.join(os.listdir(profile_path)) # NOTE: For historic reasons this is not actually compressed # even though the tar_path says ".tar.gz" check_subprocess( 'tar cf {path} -C {profile} {file_list}'.format( path=tar_path, profile=profile_path, file_list=file_list)) logging.debug('Copied profile to %s', tar_path)
def ensure_gist_repo(git, input_dir, gist_url): index = gist_url.rfind("/") if index < 0: index = gist_url.rfind(":") # ssh gist gist_id = gist_url[index + 1 :] git_dir = os.path.join(input_dir, gist_id) if not os.path.exists(git_dir): logging.debug("Cloning gist from %s", gist_url) ensure_dir_exists(os.path.dirname(git_dir)) git_run_with_retries(git, os.path.dirname(git_dir), "clone " + gist_url) else: logging.debug('Updating gist in "%s"', git_dir) git_run_with_retries(git, git_dir, "fetch origin main") git_run_with_retries(git, git_dir, "checkout main") git_run_with_retries(git, git_dir, "reset --hard origin/main") return git_dir
def __build_from_live_server(self, repository): """Implements CommandProcessor interface.""" docs_url_path = SWAGGER_URL_PATHS[repository.name] env = dict(os.environ) port = unused_port() env['SERVER_PORT'] = str(port) base_url = 'http://localhost:' + str(port) gate_logfile = self.get_logfile_path(repository.name + '-apidocs-server') logging.info('Starting up prototype %s so we can extract docs from it.' ' We will log this instance to %s', repository.name, gate_logfile) boot_run_cmd = './gradlew' # default will run ensure_dir_exists(os.path.dirname(gate_logfile)) gate_logstream = open(gate_logfile, 'w') process = start_subprocess( boot_run_cmd, stream=gate_logstream, stdout=gate_logstream, cwd=repository.git_dir, env=env) max_wait_secs = self.options.max_wait_secs_startup # pylint: disable=broad-except try: logging.info('Waiting up to %s secs for %s to be ready on port %d', max_wait_secs, repository.name, port) self.wait_for_url(base_url + '/health', max_wait_secs) json_path = os.path.join(self.get_output_dir(), 'docs.json') self.__generate_json_from_url( repository, base_url + '/' + docs_url_path, json_path) self.build_swagger_docs(repository, json_path) finally: try: gate_logstream.flush() gate_logstream.write( '\n' + log_timestring() + ' ***** buildtool is killing subprocess *****\n') logging.info('Killing %s subprocess %s now that we are done with it', repository.name, process.pid) process.kill() wait_subprocess(process) gate_logstream.close() except Exception as ex: maybe_log_exception( self.name, ex, 'Ignoring exception while stopping {name} subprocess {pid}.' .format(name=repository.name, pid=process.pid))
def ensure_gist_repo(git, input_dir, gist_url): index = gist_url.rfind('/') if index < 0: index = gist_url.rfind(':') # ssh gist gist_id = gist_url[index + 1:] git_dir = os.path.join(input_dir, gist_id) if not os.path.exists(git_dir): logging.debug('Cloning gist from %s', gist_url) ensure_dir_exists(os.path.dirname(git_dir)) git_run_with_retries(git, os.path.dirname(git_dir), 'clone ' + gist_url) else: logging.debug('Updating gist in "%s"', git_dir) git_run_with_retries(git, git_dir, 'fetch origin master') git_run_with_retries(git, git_dir, 'checkout master') git_run_with_retries(git, git_dir, 'reset --hard origin/master') return git_dir
def build_swagger_docs(self, repository, docs_url): """Build the API from the swagger endpoint.""" if repository.name != 'gate': raise_and_log_error( UnexpectedError('Repo "%s" != "gate"' % repository.name)) docs_dir = self.get_output_dir() ensure_dir_exists(docs_dir) docs_path = os.path.join(docs_dir, 'docs.json') logging.info('Generating swagger docs for %s', repository.name) check_subprocess('curl -s {url} -o {docs_path}' .format(url=docs_url, docs_path=docs_path)) check_subprocess( 'java -jar {jar_path} generate -i {docs_path} -l html2' ' -o {output_dir} -t {templates_directory}' .format(jar_path=self.options.swagger_codegen_cli_jar_path, docs_path=docs_path, output_dir=docs_dir, templates_directory=self.__templates_directory)) logging.info('Writing docs to directory %s', docs_dir)
def __collect_halconfig_files(self, repository): """Gets the component config files and writes them into the output_dir.""" name = repository.name if (name not in SPINNAKER_BOM_REPOSITORY_NAMES or name in ['spin']): logging.debug('%s does not use config files -- skipping', name) return if name == 'spinnaker-monitoring': config_root = os.path.join( repository.git_dir, 'spinnaker-monitoring-daemon') else: config_root = repository.git_dir service_name = self.scm.repository_name_to_service_name(repository.name) target_dir = os.path.join(self.get_output_dir(), 'halconfig', service_name) ensure_dir_exists(target_dir) config_path = os.path.join(config_root, 'halconfig') logging.info('Copying configs from %s...', config_path) for profile in os.listdir(config_path): profile_path = os.path.join(config_path, profile) if os.path.isfile(profile_path): shutil.copyfile(profile_path, os.path.join(target_dir, profile)) logging.debug('Copied profile to %s', profile_path) elif not os.path.isdir(profile_path): logging.warning('%s is neither file nor directory -- ignoring', profile_path) continue else: tar_path = os.path.join( target_dir, '{profile}.tar.gz'.format(profile=profile)) file_list = ' '.join(os.listdir(profile_path)) # NOTE: For historic reasons this is not actually compressed # even though the tar_path says ".tar.gz" check_subprocess( 'tar cf {path} -C {profile} {file_list}'.format( path=tar_path, profile=profile_path, file_list=file_list)) logging.debug('Copied profile to %s', tar_path)
def check_subprocesses_to_logfile(what, logfile, cmds, append=False, **kwargs): """Wrapper around check_subprocess that logs output to a logfile. Args: what: [string] For logging purposes, what is the command for. logfile: [path] The logfile to write to. cmds: [list of string] A list of commands to run. append: [boolean] Open the log file as append if true, write new default. kwargs: [kwargs] Additional keyword arguments to pass to check_subprocess. """ mode = "a" if append else "w" how = "Appending" if append else "Logging" logging.info("%s %s to %s", how, what, logfile) ensure_dir_exists(os.path.dirname(logfile)) with io.open(logfile, mode, encoding="utf-8") as stream: try: check_subprocess_sequence(cmds, stream=stream, embed_errors=False, **kwargs) except Exception as ex: logging.error("%s failed. Log file [%s] follows:", what, logfile) import traceback traceback.print_exc() with io.open(logfile, "r", encoding="utf-8") as readagain: output = readagain.read() log_embedded_output(logging.ERROR, logfile, output) logging.error( "Caught exception %s\n%s failed. See embedded logfile above", ex, what ) ensure_dir_exists(ERROR_LOGFILE_DIR) error_path = os.path.join("errors", os.path.basename(logfile)) logging.info("Copying error log file to %s", error_path) with io.open(error_path, "w", encoding="utf-8") as f: f.write(output) f.write("\n--------\n") f.write("Exeception caught in parent process:\n%s" % ex) raise
def _do_repository(self, repository): generate_path = self.options.generate_swagger_path if not generate_path: self.__build_from_live_server(repository) return swagger_dir = os.path.dirname(generate_path) generate_logfile = self.get_logfile_path( repository.name + '-generate_swagger') check_subprocesses_to_logfile( 'Extracting API to JSON', generate_logfile, [generate_path], cwd=repository.git_dir) docs_dir = self.get_output_dir() ensure_dir_exists(docs_dir) json_path = os.path.join(docs_dir, 'docs.json') swagger_output_path = os.path.join( repository.git_dir, swagger_dir, 'swagger.json') with open(swagger_output_path, 'r') as stream: content = stream.read() with open(json_path, 'w') as stream: stream.write(content) self.build_swagger_docs(repository, json_path)
def _do_repository(self, repository): generate_path = self.options.generate_swagger_path if not generate_path: self.__build_from_live_server(repository) return docs_url_path = SWAGGER_URL_PATHS[repository.name] swagger_dir = os.path.dirname(generate_path) generate_logfile = self.get_logfile_path(repository.name + '-generate_swagger') check_subprocesses_to_logfile('Extracting API to JSON', generate_logfile, [generate_path], cwd=repository.git_dir) docs_dir = self.get_output_dir() ensure_dir_exists(docs_dir) json_path = os.path.join(docs_dir, 'docs.json') swagger_output_path = os.path.join(repository.git_dir, swagger_dir, 'swagger.json') with open(swagger_output_path, 'r') as stream: content = stream.read() with open(json_path, 'w') as stream: stream.write(content) self.build_swagger_docs(repository, json_path)