def _get_nebula_repository_version(self, name):
"""Get the GitSummary for the named repo."""
if not self.__nebula_repository_version.get(name):
options = self.options
summary_path = os.path.join(options.scratch_dir, name,
'{name}-summary.yml'.format(name=name))
if os.path.exists(summary_path):
with open(summary_path, 'r') as stream:
data = yaml.load(stream.read())
self.__nebula_repository_version[name] = data['version']
else:
logging.debug(
'Forcing a repository summary because there is no %s',
summary_path)
git_dir = self.source_code_manager.get_local_repository_path(
name)
original_repo_path = os.path.abspath(git_dir)
summary = self.git.collect_repository_summary(
original_repo_path)
summary_yaml = summary.to_yaml(with_commit_messages=False)
self.__nebula_repository_version[name] = summary.version
ensure_dir_exists(os.path.dirname(summary_path))
with open(summary_path, 'w') as stream:
stream.write(summary_yaml)
return self.__nebula_repository_version[name]
def clone_repository_to_path(self,
remote_url,
git_dir,
upstream_url=None,
commit=None,
branch=None,
default_branch=None):
"""Clone the remote repository at the given commit or branch.
If requesting a branch and it is not found, then settle for the default
branch, if one was explicitly specified.
"""
# pylint: disable=too-many-arguments
if (commit != None) and (branch != None):
raise ValueError(
'At most one of commit or branch can be specified.')
logging.debug('Begin cloning %s', remote_url)
parent_dir = os.path.dirname(git_dir)
ensure_dir_exists(parent_dir)
clone_command = 'clone ' + remote_url
if branch:
branches = [branch]
if default_branch:
branches.append(default_branch)
self.__check_clone_branch(remote_url, parent_dir, clone_command,
branches)
else:
self.check_git(parent_dir, clone_command)
logging.info('Cloned %s into %s', remote_url, parent_dir)
logging.info(
'Renaming origin remote and adding a decoy to trick nebula.')
self.check_git(git_dir,
'remote rename origin ' + self.ORIGIN_REMOTE_NAME)
decoy_origin_dir = self.__make_decoy_origin(os.path.basename(git_dir))
self.check_git(
git_dir, 'remote add origin ' + os.path.abspath(decoy_origin_dir))
if commit:
self.check_git(git_dir, 'checkout -q ' + commit, echo=True)
if upstream_url and not self.is_same_repo(upstream_url, remote_url):
logging.debug('Adding upstream %s with disabled push',
upstream_url)
self.check_git(git_dir, 'remote add upstream ' + upstream_url)
which = ('upstream' if upstream_url
and not self.is_same_repo(upstream_url, remote_url) else
self.ORIGIN_REMOTE_NAME)
self.check_git(
git_dir,
'remote set-url --push {which} disabled'.format(which=which))
logging.debug('Finished cloning %s', remote_url)
def _do_repository(self, repository):
"""Implements CommandProcessor interface."""
docs_url_path = SWAGGER_URL_PATHS[repository.name]
scm = self.source_code_manager
git_dir = scm.get_local_repository_path(repository.name)
if self.options.fetch_bom_path or self.options.fetch_bom_version:
scm.pull_source_from_bom(repository.name, git_dir, self.__bom)
env = dict(os.environ)
port = unused_port()
env['SERVER_PORT'] = str(port)
base_url = 'http://localhost:{port}'.format(port=port)
gate_logfile = determine_logfile_path(self.options, repository.name,
'apidocs')
logging.info(
'Starting up prototype %s so we can extract docs from it.'
' We will log this instance to %s', repository.name, gate_logfile)
boot_run_cmd = './gradlew bootRun'
ensure_dir_exists(os.path.dirname(gate_logfile))
gate_logstream = open(gate_logfile, 'w')
process = start_subprocess(boot_run_cmd,
stream=gate_logstream,
stdout=gate_logstream,
cwd=git_dir,
env=env)
max_wait_secs = self.options.max_wait_secs_startup
# pylint: disable=broad-except
try:
logging.info('Waiting up to %s secs for %s to be ready on port %d',
max_wait_secs, repository.name, port)
self.wait_for_url(base_url + '/health', max_wait_secs)
self.generate_swagger_docs(
repository, '{base}/{path}'.format(base=base_url,
path=docs_url_path))
finally:
try:
gate_logstream.flush()
gate_logstream.write(
'\n{time} ***** buildtool is killing subprocess *****\n'.
format(time=timestring()))
logging.info(
'Killing %s subprocess %s now that we are done with it',
repository.name, process.pid)
process.kill()
wait_subprocess(process)
gate_logstream.close()
except Exception as ex:
maybe_log_exception(
self.name, ex,
'Ignoring exception while stopping {name} subprocess {pid}.'
.format(name=repository.name, pid=process.pid))
def clone_repository_to_path(self,
origin_url,
git_dir,
upstream_url=None,
commit=None,
branch=None,
default_branch=None):
"""Clone the remote repository at the given commit or branch.
If requesting a branch and it is not found, then settle for the default
branch, if one was explicitly specified.
"""
# pylint: disable=too-many-arguments
if (commit != None) and (branch != None):
raise ValueError(
'At most one of commit or branch can be specified.')
logging.debug('Begin cloning %s', origin_url)
parent_dir = os.path.dirname(git_dir)
ensure_dir_exists(parent_dir)
git_command = 'git -C "{parent_dir}" clone {origin_url}'.format(
parent_dir=parent_dir, origin_url=origin_url)
if branch:
branches = [branch]
if default_branch:
branches.append(default_branch)
self.__check_clone_branch(origin_url, git_command, branches)
else:
check_subprocess(git_command)
logging.info('Cloned %s into %s', origin_url, parent_dir)
if commit:
check_subprocess('git -C "{dir}" checkout {commit} -q'.format(
dir=git_dir, commit=commit),
echo=True)
if upstream_url and not self.same_repo(upstream_url, origin_url):
logging.debug('Adding upstream %s with disabled push',
upstream_url)
check_subprocess(
'git -C "{dir}" remote add upstream {upstream_url}'.format(
dir=git_dir, upstream_url=upstream_url))
which = ('upstream' if upstream_url
and not self.same_repo(upstream_url, origin_url) else
'origin')
check_subprocess(
'git -C "{dir}" remote set-url --push {which} disabled'.format(
dir=git_dir, which=which))
logging.debug('Finished cloning %s', origin_url)
def __make_decoy_origin(self, repo_name):
"""Create a empty repository as a decoy to trick nebula.
When nebula insists on side effects to the "origin" repo, it will
munge this one leaving the real origin alone.
"""
decoy_path = os.path.join(self.options.scratch_dir, 'nebula_decoys',
repo_name)
ensure_dir_exists(decoy_path)
if not os.path.exists(os.path.join(decoy_path, '.git')):
logging.debug('Creating nebula decoy repository at %s', decoy_path)
self.check_git(decoy_path, 'init')
return decoy_path
def __collect_halconfig_files(self, repository):
"""Gets the component config files and writes them into the scratch_path."""
name = repository.name
if (name not in SPINNAKER_RUNNABLE_REPOSITORIES.keys()
and name not in ['spinnaker-monitoring']):
logging.debug('%s does not use config files -- skipping', name)
return
git_dir = self.source_code_manager.get_local_repository_path(name)
if name == 'spinnaker-monitoring':
config_root = os.path.join(git_dir, 'spinnaker-monitoring-daemon')
else:
config_root = git_dir
options = self.options
target_dir = os.path.join(options.scratch_dir, name, 'halconfig')
ensure_dir_exists(target_dir)
config_path = os.path.join(config_root, 'halconfig')
logging.info('Copying configs from %s...', config_path)
for profile in os.listdir(config_path):
profile_path = os.path.join(config_path, profile)
if os.path.isfile(profile_path):
shutil.copyfile(profile_path,
os.path.join(target_dir, profile))
logging.debug('Copied profile to %s', profile_path)
elif not os.path.isdir(profile_path):
logging.warning('%s is neither file nor directory -- ignoring',
profile_path)
continue
else:
tar_path = os.path.join(
target_dir, '{profile}.tar.gz'.format(profile=profile))
file_list = ' '.join(os.listdir(profile_path))
# NOTE: For historic reasons this is not actually compressed
# even though the tar_path says ".tar.gz"
check_subprocess(
'tar cf {path} -C {profile} {file_list}'.format(
path=tar_path,
profile=profile_path,
file_list=file_list))
logging.debug('Copied profile to %s', tar_path)
def maybe_pull_repository_source(self,
repository,
bom_version=None,
bom_path=None,
git_branch=None,
default_branch=None):
"""Pull the source as specified, if it was specified."""
# pylint: disable=too-many-arguments
pull_version = 1 if bom_version else 0
pull_path = 1 if bom_path else 0
pull_branch = 1 if git_branch else 0
have = pull_version + pull_path + pull_branch
if have == 0:
logging.debug('No source pull requests to process.')
return
if have > 1:
raise ValueError('Ambiguous source code requests.')
if default_branch and not git_branch:
raise ValueError('A default_branch requires a git_branch')
git_dir = self.get_local_repository_path(repository.name)
if os.path.exists(git_dir):
message = '%s already exists. Cannot pull source.' % git_dir
logging.error(message)
raise ValueError(message)
logging.info('Pulling %s to %s', repository.url, git_dir)
ensure_dir_exists(os.path.dirname(git_dir))
if git_branch:
self.git.clone_repository_to_path(repository.url,
git_dir,
branch=git_branch,
default_branch=default_branch)
else:
if bom_version:
bom = self.bom_from_version(bom_version)
else:
bom = self.bom_from_path(bom_path)
self.pull_source_from_bom(repository.name, git_dir, bom)
def generate_swagger_docs(self, repository, docs_url):
"""Generate the API from the swagger endpoint."""
options = self.options
named_scratch_dir = os.path.join(options.scratch_dir, repository.name)
docs_dir = os.path.abspath(os.path.join(named_scratch_dir, 'apidocs'))
docs_path = os.path.join(docs_dir, 'docs.json')
ensure_dir_exists(docs_dir)
logging.info('Generating swagger docs for %s', repository.name)
check_subprocess('curl -s {url} -o {docs_path}'.format(
url=docs_url, docs_path=docs_path))
check_subprocess(
'java -jar {jar_path} generate -i {docs_path} -l html2'
' -o {scratch} -t {templates_directory}'.format(
jar_path=options.swagger_codegen_cli_jar_path,
docs_path=docs_path,
scratch=docs_dir,
templates_directory=self.__templates_directory))
logging.info('Writing docs to directory %s', docs_dir)