def next(self, at_index):
"""Returns the next SemanticVersion from this when bumping up.
Args:
at_index: [int] The component *_INDEX to bump at.
"""
if at_index is None:
raise_and_log_error(UnexpectedError("Invalid index={0}".format(at_index)))
major = self.major
minor = self.minor
patch = self.patch
if at_index == self.PATCH_INDEX:
patch += 1
else:
patch = 0
if at_index == self.MINOR_INDEX:
minor += 1
elif at_index == self.MAJOR_INDEX:
minor = 0
major += 1
else:
raise_and_log_error(
UnexpectedError("Invalid index={0}".format(at_index))
)
return SemanticVersion(self.series_name, major, minor, patch)
def validate_test_requirements(self, test_name, spec, metric_labels):
"""Determine whether or not the test requirements are satisfied.
If not, record the reason a skip or failure.
This may throw exceptions, which are immediate failure.
Args:
test_name: [string] The name of the test.
spec: [dict] The profile specification containing requirements.
This argument will be pruned as values are consumed from it.
Returns:
True if requirements are satisifed, False if not.
"""
if not 'api' in spec:
raise_and_log_error(
UnexpectedError('Test "{name}" is missing an "api" spec.'.format(
name=test_name)))
requires = spec.pop('requires', {})
configuration = requires.pop('configuration', {})
our_config = vars(self.options)
for key, value in configuration.items():
if key not in our_config:
message = ('Unknown configuration key "{0}" for test "{1}"'
.format(key, test_name))
raise_and_log_error(ConfigError(message))
if value != our_config[key]:
reason = ('Skipped test {name} because {key}={want} != {have}'
.format(name=test_name, key=key,
want=value, have=our_config[key]))
with self.__lock:
self.__record_skip_test(test_name, reason,
'IncompatableConfig', metric_labels)
return False
services = set(replace_ha_services(
requires.pop('services', []), self.options))
services.add(self.__replace_ha_api_service(
spec.pop('api'), self.options))
if requires:
raise_and_log_error(
ConfigError('Unexpected fields in {name}.requires: {remaining}'
.format(name=test_name, remaining=requires)))
if spec:
raise_and_log_error(
ConfigError('Unexpected fields in {name} specification: {remaining}'
.format(name=test_name, remaining=spec)))
def wait_on_services(services):
thread_pool = ThreadPool(len(services))
thread_pool.map(self.wait_on_service, services)
thread_pool.terminate()
self.__deployer.metrics.track_and_time_call(
'WaitingOnServiceAvailability',
metric_labels, self.__deployer.metrics.default_determine_outcome_labels,
wait_on_services, services)
return True
def prepare_local_repository_files(self, repository):
if repository.name != SPINNAKER_GITHUB_IO_REPOSITORY_NAME:
raise_and_log_error(UnexpectedError('Got "%s"' % repository.name))
timestamp = '{:%Y-%m-%d %H:%M:%S +0000}'.format(
datetime.datetime.utcnow())
version = self.options.spinnaker_version
changelog_filename = '{version}-changelog.md'.format(version=version)
target_path = os.path.join(repository.git_dir, '_changelogs',
changelog_filename)
major, minor, _ = version.split('.')
logging.debug('Adding changelog file %s', target_path)
with open(target_path, 'w') as f:
# pylint: disable=trailing-whitespace
header = textwrap.dedent("""\
---
title: Version {major}.{minor}
changelog_title: Version {version}
date: {timestamp}
tags: changelogs {major}.{minor}
version: {version}
---
""".format(version=version,
timestamp=timestamp,
major=major,
minor=minor))
f.write(header)
f.write('<script src="%s.js"/>' % self.options.changelog_gist_url)
return [target_path]
def make_repository_spec(self, name, **kwargs):
"""Create GitRepositorySpec based on the name and configuration.
Args:
git_dir: if supplied then use it, otherwise default under the root path.
origin: if supplied then use it, even if None. Otherwise default
upstream: if supplied then use it, even if None. Otherwise default.
kwargs: Additional repository attributes
"""
git_dir = kwargs.pop("git_dir", os.path.join(self.__root_source_dir, name))
origin = kwargs.pop("origin", self.AUTO)
upstream = kwargs.pop("upstream", self.AUTO)
if origin == self.AUTO:
origin = self.determine_origin(name)
if os.path.exists(git_dir):
logging.info("Confirming existing %s matches expectations", git_dir)
existing = self.__git.determine_git_repository_spec(git_dir)
if existing.origin not in [origin, self.__git.determine_pull_url(origin)]:
raise_and_log_error(
UnexpectedError(
'Repository "{dir}" origin="{have}" expected="{want}"'.format(
dir=git_dir, have=existing.origin, want=origin
)
)
)
if upstream == self.AUTO:
upstream = self.determine_upstream_url(name)
return GitRepositorySpec(
name, origin=origin, upstream=upstream, git_dir=git_dir, **kwargs
)
def make_repository_spec(self, name, **kwargs):
"""Create GitRepositorySpec based on the name and configuration.
Args:
git_dir: if supplied then use it, otherwise default under the root path.
origin: if supplied then use it, even if None. Otherwise default
upstream: if supplied then use it, even if None. Otherwise default.
"""
git_dir = kwargs.pop('git_dir', os.path.join(self.__root_source_dir, name))
origin = kwargs.pop('origin', self.AUTO)
upstream = kwargs.pop('upstream', self.AUTO)
check_kwargs_empty(kwargs)
if origin == self.AUTO:
origin = self.determine_origin(name)
if os.path.exists(git_dir):
logging.info('Confirming existing %s matches expectations', git_dir)
existing = self.__git.determine_git_repository_spec(git_dir)
if existing.origin != origin:
raise_and_log_error(
UnexpectedError(
'Repository "{dir}" origin="{have}" expected="{want}"'.format(
dir=git_dir, have=existing.origin, want=origin)))
if upstream == self.AUTO:
upstream = self.determine_upstream_url(name)
return GitRepositorySpec(
name, origin=origin, upstream=upstream, git_dir=git_dir)
def determine_source_repositories(self):
"""Determine which repositories are available to this SCM."""
raise_and_log_error(
UnexpectedError(
self.__class__.__name__ + ": Should only be applicable to BomSCM",
cause="NotReachable",
)
)
def get_repository_service_build_version(self, repository):
if not self.__bom:
raise_and_log_error(UnexpectedError('Missing bom', cause='NotReachable'))
service_name = self.repository_name_to_service_name(repository.name)
service_entry = self.__bom.get('services', {}).get(service_name, {})
if not service_entry:
raise_and_log_error(ConfigError('BOM missing service %s' % service_name))
return service_entry['version']
def determine_repository_version(self, repository):
service_name = self.repository_name_to_service_name(repository.name)
if not service_name in self.__bom['services'].keys():
raise_and_log_error(
UnexpectedError('"%s" is not a BOM repo' % service_name))
service = check_bom_service(self.__bom, service_name)
version = service['version'][:service['version'].find('-')]
return version
def check_repository_is_current(self, repository):
branch = self.options.git_branch or 'master'
git_dir = repository.git_dir
have_branch = self.git.query_local_repository_branch(git_dir)
if have_branch != branch:
raise_and_log_error(
UnexpectedError('"%s" is at the wrong branch "%s"' %
(git_dir, branch)))
return True
def check_repository_is_current(self, repository):
git_dir = repository.git_dir
commit = repository.commit_or_none()
if commit is not None:
have_commit = self.git.query_local_repository_commit_id(git_dir)
if have_commit != commit:
raise_and_log_error(
UnexpectedError(
'"%s" is at the wrong commit "%s" vs "%s"' %
(git_dir, have_commit, commit)))
return True
branch = self.options.git_branch or 'master'
have_branch = self.git.query_local_repository_branch(git_dir)
if have_branch != branch:
raise_and_log_error(
UnexpectedError('"%s" is at the wrong branch "%s" vs "%s"' %
(git_dir, have_branch, branch)))
return True
def query_commit_at_tag(self, git_dir, tag):
"""Return the commit for the given tag, or None if tag is not known."""
retcode, stdout = self.run_git(git_dir, 'show-ref -- ' + tag)
if retcode != 0:
return None
lines = stdout.split('\n')
if len(lines) != 1:
raise_and_log_error(
UnexpectedError('"{tag}" -> "{msg}"'.format(tag=tag, msg=stdout)))
return stdout.split(' ')[0]
def check_repository_is_current(self, repository):
git_dir = repository.git_dir
service_name = self.repository_name_to_service_name(repository.name)
have_commit = self.git.query_local_repository_commit_id(git_dir)
bom_commit = check_bom_service(self.__bom, service_name)['commit']
if have_commit != bom_commit:
raise_and_log_error(
UnexpectedError('"%s" is at the wrong commit "%s"' %
(git_dir, bom_commit)))
return True
def prepare_local_repository_files(self, repository):
if repository.name != SPINNAKER_GITHUB_IO_REPOSITORY_NAME:
raise_and_log_error(UnexpectedError('Got "%s"' % repository.name))
updated_files = []
new_version = self.write_new_version(repository)
updated_files.append(new_version)
old_version = self.deprecate_prior_version(repository)
if old_version is not None:
updated_files.append(old_version)
return updated_files
def check_source_info(self, repository):
"""Ensure cached source info is consistent with current repository."""
logging.debug('Checking that cached commit is consistent with %s',
repository.git_dir)
info = self.lookup_source_info(repository)
commit = self.__git.query_local_repository_commit_id(repository.git_dir)
cached_commit = info.summary.commit_id
if cached_commit != commit:
raise_and_log_error(
UnexpectedError(
'Cached commit {cache} != current commit {id} in {dir}'.format(
cache=cached_commit, id=commit, dir=repository.git_dir)))
return info
def make(tag):
"""Create a new SemanticVersion from the given tag instance.
Args:
tag: [string] in the form <series_name>-<major>.<minor>.<patch>
"""
match = SemanticVersion.SEMVER_MATCHER.match(tag)
if match is None:
raise_and_log_error(UnexpectedError('Malformed tag "%s"' % tag))
# Keep first group as a string, but use integers for the component parts
return SemanticVersion(match.group(1),
*[int(num) for num in match.groups()[1:]])
def determine_git_repository_spec(self, git_dir):
"""Infer GitRepositorySpec from a local git repository."""
git_text = self.check_run(git_dir, 'remote -v')
remote_urls = {
match.group(1): match.group(2)
for match in re.finditer(r'(\w+)\s+(\S+)\s+\(fetch\)', git_text)
}
origin_url = remote_urls.get('origin')
if not origin_url:
raise_and_log_error(
UnexpectedError('{0} has no remote "origin"'.format(git_dir)))
return GitRepositorySpec(os.path.basename(git_dir),
git_dir=git_dir,
origin=origin_url,
upstream=remote_urls.get('upstream'))
def build_swagger_docs(self, repository, json_path):
"""Build the API from the swagger endpoint."""
if repository.name != 'gate':
raise_and_log_error(
UnexpectedError('Repo "%s" != "gate"' % repository.name))
docs_dir = os.path.dirname(json_path)
check_subprocess(
'java -jar {jar_path} generate -i {json_path} -l html2'
' -o {output_dir} -t {templates_directory}'.format(
jar_path=self.options.swagger_codegen_cli_jar_path,
json_path=json_path,
output_dir=docs_dir,
templates_directory=self.__templates_directory))
logging.info('Writing docs to directory %s', docs_dir)
def ingest_bom(self, line):
"""Function to ingest a single bom into the result map."""
bom = self.load_bom_from_url(line)
if not bom:
return
try:
if bom['version'] + '.yml' != line[line.rfind('/') + 1:]:
message = 'BOM version "%s" != filename "%s"' % (bom['version'], line)
self.__bad_files[self.url_to_bom_name(line.strip())] = message
logging.warning(message)
raise_and_log_error(UnexpectedError(message))
self.analyze_bom(bom)
except Exception as ex:
self.__bad_files[self.url_to_bom_name(line.strip())] = ex.message
maybe_log_exception('analyze_bom', ex,
action_msg='Skipping %s' % line)
def ensure_git_path(self, repository, **kwargs):
"""Make sure repository path is consistent with BOM."""
check_kwargs_empty(kwargs)
service_name = self.repository_name_to_service_name(repository.name)
if not service_name in self.__bom['services'].keys():
raise_and_log_error(
UnexpectedError('"%s" is not a BOM repo' % service_name))
git_dir = repository.git_dir
have_git_dir = os.path.exists(git_dir)
service = check_bom_service(self.__bom, service_name)
commit_id = service['commit']
if not have_git_dir:
self.git.clone_repository_to_path(repository, commit=commit_id)
def register(self, registry, subparsers, defaults):
"""Registers a command factory.
Args:
registry: [dict] The registry to add to, keyed by command name.
subparsers: [ArgumentParser subparsers] for adding command arguments
defaults: [dict] optional default values for command arguments
"""
factory = self
name = factory.name
if name in registry.keys():
raise_and_log_error(
UnexpectedError(
'CommandFactory "{name}" already exists.'.format(name=name)))
factory.add_argparser(subparsers, defaults)
registry[name] = factory
def patchable(self):
"""Return True if the changes in this repository is only a patch release."""
previous_parts = self.prev_version.split('.')
current_parts = self.version.split('.')
if len(previous_parts) != 3:
raise_and_log_error(
ConfigError('Previous version %s is not X.Y.Z' % self.prev_version))
if len(current_parts) != 3:
raise_and_log_error(
ConfigError('Version %s is not X.Y.Z' % self.version))
if previous_parts[:2] != current_parts[:2]:
return False
if int(previous_parts[2]) != int(current_parts[2]) - 1:
raise_and_log_error(
UnexpectedError(
'Unexpected version sequence {prev} to {current}'.format(
prev=self.prev_version, current=self.version)))
return True
def build_swagger_docs(self, repository, docs_url):
"""Build the API from the swagger endpoint."""
if repository.name != 'gate':
raise_and_log_error(
UnexpectedError('Repo "%s" != "gate"' % repository.name))
docs_dir = self.get_output_dir()
ensure_dir_exists(docs_dir)
docs_path = os.path.join(docs_dir, 'docs.json')
logging.info('Generating swagger docs for %s', repository.name)
check_subprocess('curl -s {url} -o {docs_path}'
.format(url=docs_url, docs_path=docs_path))
check_subprocess(
'java -jar {jar_path} generate -i {docs_path} -l html2'
' -o {output_dir} -t {templates_directory}'
.format(jar_path=self.options.swagger_codegen_cli_jar_path,
docs_path=docs_path, output_dir=docs_dir,
templates_directory=self.__templates_directory))
logging.info('Writing docs to directory %s', docs_dir)
def __determine_repo_install_args(self, repository):
"""Determine --spinnaker_dev-github_[owner|user] args for install script."""
options = self.options
branch = options.git_branch
owner = ('spinnaker' if options.github_owner in ('default', 'upstream')
else options.github_owner)
git_dir = os.path.dirname(__file__)
if not branch:
branch = GitRunner(options).query_local_repository_branch(git_dir)
if not owner:
url = repository.origin
match = re.search('github.com/([^/]+)/', url)
if not match:
raise_and_log_error(
UnexpectedError('Cannot determine owner from url=%s' % url,
cause='BadUrl'))
owner = match.group(1)
return [
'--spinnaker_dev_github_owner', owner,
'--spinnaker_dev_github_branch', branch
]
def _do_repository(self, repository):
if repository.name != SPINNAKER_IO_REPOSITORY_NAME:
raise_and_log_error(UnexpectedError('Got "%s"' % repository.name))
base_branch = "master"
self.scm.ensure_git_path(repository, branch=base_branch)
version = self.options.spinnaker_version
if self.options.git_allow_publish_master_branch:
branch_flag = ""
head_branch = "master"
else:
branch_flag = "-B"
head_branch = version + "-changelog"
files_added = self.prepare_local_repository_files(repository)
git_dir = repository.git_dir
message = "doc(changelog): Spinnaker Version " + version
local_git_commands = [
# These commands are accomodating to a branch already existing
# because the branch is on the version, not build. A rejected
# build for some reason that is re-tried will have the same version
# so the branch may already exist from the earlier attempt.
"fetch origin " + base_branch,
"checkout " + base_branch,
"checkout {flag} {branch}".format(flag=branch_flag, branch=head_branch),
"add " + " ".join([os.path.abspath(path) for path in files_added]),
]
logging.debug(
'Commiting changes into local repository "%s" branch=%s',
repository.git_dir,
head_branch,
)
git = self.git
git.check_run_sequence(git_dir, local_git_commands)
git.check_commit_or_no_changes(git_dir, '-m "{msg}"'.format(msg=message))
logging.info('Pushing branch="%s" into "%s"', head_branch, repository.origin)
git.push_branch_to_origin(git_dir, branch=head_branch)
def load_bom(options):
"""Helper function for initializing the BOM if one was specified."""
bom_path = options.bom_path if hasattr(options, 'bom_path') else None
bom_version = (options.bom_version
if hasattr(options, 'bom_version')
else None)
have_bom_path = 1 if bom_path else 0
have_bom_version = 1 if bom_version else 0
if have_bom_path + have_bom_version != 1:
raise_and_log_error(
ConfigError('Expected exactly one of: "bom_path", or "bom_version"'))
if bom_path:
check_path_exists(bom_path, why='options.bom_path')
return BomSourceCodeManager.bom_from_path(bom_path)
if bom_version:
logging.debug('Retrieving bom version %s', bom_version)
return HalRunner(options).retrieve_bom_version(bom_version)
raise_and_log_error(UnexpectedError('Not reachable', cause='NotReachable'))
def make(entry):
"""Create a new CommitMessage from an individual entry"""
match = CommitMessage._MEDIUM_PRETTY_COMMIT_MATCHER.match(entry)
if match is None:
raise_and_log_error(
UnexpectedError('Unexpected commit entry {0}'.format(entry)))
text = entry[match.end(3):]
# strip trailing spaces on each line
lines = [line.rstrip() for line in text.split('\n')]
# remove blank lines from beginning and end of text
while lines and not lines[0]:
del lines[0]
while lines and not lines[-1]:
del lines[-1]
# new string may have initial spacing but no leading/trailing blank lines.
text = '\n'.join(lines)
return CommitMessage(match.group(1), match.group(2), match.group(3), text)
def __determine_repo_install_args(self, repository):
"""Determine --spinnaker_dev-github_[owner|user] args for install script."""
options = self.options
branch = options.git_branch
owner = ("spinnaker" if options.github_owner in ("default", "upstream")
else options.github_owner)
git_dir = os.path.dirname(__file__)
if not branch:
branch = GitRunner(options).query_local_repository_branch(git_dir)
if not owner:
url = repository.origin
match = re.search("github.com/([^/]+)/", url)
if not match:
raise_and_log_error(
UnexpectedError("Cannot determine owner from url=%s" % url,
cause="BadUrl"))
owner = match.group(1)
return [
"--spinnaker_dev_github_owner",
owner,
"--spinnaker_dev_github_branch",
branch,
]
def prepare_local_repository_files(self, repository):
if repository.name != SPINNAKER_GITHUB_IO_REPOSITORY_NAME:
raise_and_log_error(UnexpectedError('Got "%s"' % repository.name))
with open(self.__markdown_path) as f:
detail = f.read()
# Use the original capture time
utc = datetime.datetime.fromtimestamp(
os.path.getmtime(self.__markdown_path))
timestamp = '{:%Y-%m-%d %H:%M:%S %Z}'.format(utc)
version = self.options.spinnaker_version
changelog_filename = '{version}-changelog.md'.format(version=version)
target_path = os.path.join(repository.git_dir,
'_changelogs', changelog_filename)
major, minor, _ = version.split('.')
logging.debug('Adding changelog file %s', target_path)
with open(target_path, 'w') as f:
# pylint: disable=trailing-whitespace
header = textwrap.dedent(
"""\
---
title: Version {version}
date: {timestamp}
tags: changelogs {major}.{minor}
---
# Spinnaker {version}
""".format(
version=version,
timestamp=timestamp,
major=major, minor=minor))
f.write(header)
f.write(detail)
return [target_path]
def determine_origin(self, name):
"""Determine the origin to use for the given repository."""
if not self.__github_owner:
raise_and_log_error(
UnexpectedError('Not reachable', cause='NotReachable'))
return self.determine_origin_for_owner(name, self.__github_owner)
