def next(self, at_index): """Returns the next SemanticVersion from this when bumping up. Args: at_index: [int] The component *_INDEX to bump at. """ if at_index is None: raise_and_log_error(UnexpectedError("Invalid index={0}".format(at_index))) major = self.major minor = self.minor patch = self.patch if at_index == self.PATCH_INDEX: patch += 1 else: patch = 0 if at_index == self.MINOR_INDEX: minor += 1 elif at_index == self.MAJOR_INDEX: minor = 0 major += 1 else: raise_and_log_error( UnexpectedError("Invalid index={0}".format(at_index)) ) return SemanticVersion(self.series_name, major, minor, patch)
def validate_test_requirements(self, test_name, spec, metric_labels): """Determine whether or not the test requirements are satisfied. If not, record the reason a skip or failure. This may throw exceptions, which are immediate failure. Args: test_name: [string] The name of the test. spec: [dict] The profile specification containing requirements. This argument will be pruned as values are consumed from it. Returns: True if requirements are satisifed, False if not. """ if not 'api' in spec: raise_and_log_error( UnexpectedError('Test "{name}" is missing an "api" spec.'.format( name=test_name))) requires = spec.pop('requires', {}) configuration = requires.pop('configuration', {}) our_config = vars(self.options) for key, value in configuration.items(): if key not in our_config: message = ('Unknown configuration key "{0}" for test "{1}"' .format(key, test_name)) raise_and_log_error(ConfigError(message)) if value != our_config[key]: reason = ('Skipped test {name} because {key}={want} != {have}' .format(name=test_name, key=key, want=value, have=our_config[key])) with self.__lock: self.__record_skip_test(test_name, reason, 'IncompatableConfig', metric_labels) return False services = set(replace_ha_services( requires.pop('services', []), self.options)) services.add(self.__replace_ha_api_service( spec.pop('api'), self.options)) if requires: raise_and_log_error( ConfigError('Unexpected fields in {name}.requires: {remaining}' .format(name=test_name, remaining=requires))) if spec: raise_and_log_error( ConfigError('Unexpected fields in {name} specification: {remaining}' .format(name=test_name, remaining=spec))) def wait_on_services(services): thread_pool = ThreadPool(len(services)) thread_pool.map(self.wait_on_service, services) thread_pool.terminate() self.__deployer.metrics.track_and_time_call( 'WaitingOnServiceAvailability', metric_labels, self.__deployer.metrics.default_determine_outcome_labels, wait_on_services, services) return True
def prepare_local_repository_files(self, repository): if repository.name != SPINNAKER_GITHUB_IO_REPOSITORY_NAME: raise_and_log_error(UnexpectedError('Got "%s"' % repository.name)) timestamp = '{:%Y-%m-%d %H:%M:%S +0000}'.format( datetime.datetime.utcnow()) version = self.options.spinnaker_version changelog_filename = '{version}-changelog.md'.format(version=version) target_path = os.path.join(repository.git_dir, '_changelogs', changelog_filename) major, minor, _ = version.split('.') logging.debug('Adding changelog file %s', target_path) with open(target_path, 'w') as f: # pylint: disable=trailing-whitespace header = textwrap.dedent("""\ --- title: Version {major}.{minor} changelog_title: Version {version} date: {timestamp} tags: changelogs {major}.{minor} version: {version} --- """.format(version=version, timestamp=timestamp, major=major, minor=minor)) f.write(header) f.write('<script src="%s.js"/>' % self.options.changelog_gist_url) return [target_path]
def make_repository_spec(self, name, **kwargs): """Create GitRepositorySpec based on the name and configuration. Args: git_dir: if supplied then use it, otherwise default under the root path. origin: if supplied then use it, even if None. Otherwise default upstream: if supplied then use it, even if None. Otherwise default. kwargs: Additional repository attributes """ git_dir = kwargs.pop("git_dir", os.path.join(self.__root_source_dir, name)) origin = kwargs.pop("origin", self.AUTO) upstream = kwargs.pop("upstream", self.AUTO) if origin == self.AUTO: origin = self.determine_origin(name) if os.path.exists(git_dir): logging.info("Confirming existing %s matches expectations", git_dir) existing = self.__git.determine_git_repository_spec(git_dir) if existing.origin not in [origin, self.__git.determine_pull_url(origin)]: raise_and_log_error( UnexpectedError( 'Repository "{dir}" origin="{have}" expected="{want}"'.format( dir=git_dir, have=existing.origin, want=origin ) ) ) if upstream == self.AUTO: upstream = self.determine_upstream_url(name) return GitRepositorySpec( name, origin=origin, upstream=upstream, git_dir=git_dir, **kwargs )
def make_repository_spec(self, name, **kwargs): """Create GitRepositorySpec based on the name and configuration. Args: git_dir: if supplied then use it, otherwise default under the root path. origin: if supplied then use it, even if None. Otherwise default upstream: if supplied then use it, even if None. Otherwise default. """ git_dir = kwargs.pop('git_dir', os.path.join(self.__root_source_dir, name)) origin = kwargs.pop('origin', self.AUTO) upstream = kwargs.pop('upstream', self.AUTO) check_kwargs_empty(kwargs) if origin == self.AUTO: origin = self.determine_origin(name) if os.path.exists(git_dir): logging.info('Confirming existing %s matches expectations', git_dir) existing = self.__git.determine_git_repository_spec(git_dir) if existing.origin != origin: raise_and_log_error( UnexpectedError( 'Repository "{dir}" origin="{have}" expected="{want}"'.format( dir=git_dir, have=existing.origin, want=origin))) if upstream == self.AUTO: upstream = self.determine_upstream_url(name) return GitRepositorySpec( name, origin=origin, upstream=upstream, git_dir=git_dir)
def determine_source_repositories(self): """Determine which repositories are available to this SCM.""" raise_and_log_error( UnexpectedError( self.__class__.__name__ + ": Should only be applicable to BomSCM", cause="NotReachable", ) )
def get_repository_service_build_version(self, repository): if not self.__bom: raise_and_log_error(UnexpectedError('Missing bom', cause='NotReachable')) service_name = self.repository_name_to_service_name(repository.name) service_entry = self.__bom.get('services', {}).get(service_name, {}) if not service_entry: raise_and_log_error(ConfigError('BOM missing service %s' % service_name)) return service_entry['version']
def determine_repository_version(self, repository): service_name = self.repository_name_to_service_name(repository.name) if not service_name in self.__bom['services'].keys(): raise_and_log_error( UnexpectedError('"%s" is not a BOM repo' % service_name)) service = check_bom_service(self.__bom, service_name) version = service['version'][:service['version'].find('-')] return version
def check_repository_is_current(self, repository): branch = self.options.git_branch or 'master' git_dir = repository.git_dir have_branch = self.git.query_local_repository_branch(git_dir) if have_branch != branch: raise_and_log_error( UnexpectedError('"%s" is at the wrong branch "%s"' % (git_dir, branch))) return True
def check_repository_is_current(self, repository): git_dir = repository.git_dir commit = repository.commit_or_none() if commit is not None: have_commit = self.git.query_local_repository_commit_id(git_dir) if have_commit != commit: raise_and_log_error( UnexpectedError( '"%s" is at the wrong commit "%s" vs "%s"' % (git_dir, have_commit, commit))) return True branch = self.options.git_branch or 'master' have_branch = self.git.query_local_repository_branch(git_dir) if have_branch != branch: raise_and_log_error( UnexpectedError('"%s" is at the wrong branch "%s" vs "%s"' % (git_dir, have_branch, branch))) return True
def query_commit_at_tag(self, git_dir, tag): """Return the commit for the given tag, or None if tag is not known.""" retcode, stdout = self.run_git(git_dir, 'show-ref -- ' + tag) if retcode != 0: return None lines = stdout.split('\n') if len(lines) != 1: raise_and_log_error( UnexpectedError('"{tag}" -> "{msg}"'.format(tag=tag, msg=stdout))) return stdout.split(' ')[0]
def check_repository_is_current(self, repository): git_dir = repository.git_dir service_name = self.repository_name_to_service_name(repository.name) have_commit = self.git.query_local_repository_commit_id(git_dir) bom_commit = check_bom_service(self.__bom, service_name)['commit'] if have_commit != bom_commit: raise_and_log_error( UnexpectedError('"%s" is at the wrong commit "%s"' % (git_dir, bom_commit))) return True
def prepare_local_repository_files(self, repository): if repository.name != SPINNAKER_GITHUB_IO_REPOSITORY_NAME: raise_and_log_error(UnexpectedError('Got "%s"' % repository.name)) updated_files = [] new_version = self.write_new_version(repository) updated_files.append(new_version) old_version = self.deprecate_prior_version(repository) if old_version is not None: updated_files.append(old_version) return updated_files
def check_source_info(self, repository): """Ensure cached source info is consistent with current repository.""" logging.debug('Checking that cached commit is consistent with %s', repository.git_dir) info = self.lookup_source_info(repository) commit = self.__git.query_local_repository_commit_id(repository.git_dir) cached_commit = info.summary.commit_id if cached_commit != commit: raise_and_log_error( UnexpectedError( 'Cached commit {cache} != current commit {id} in {dir}'.format( cache=cached_commit, id=commit, dir=repository.git_dir))) return info
def make(tag): """Create a new SemanticVersion from the given tag instance. Args: tag: [string] in the form <series_name>-<major>.<minor>.<patch> """ match = SemanticVersion.SEMVER_MATCHER.match(tag) if match is None: raise_and_log_error(UnexpectedError('Malformed tag "%s"' % tag)) # Keep first group as a string, but use integers for the component parts return SemanticVersion(match.group(1), *[int(num) for num in match.groups()[1:]])
def determine_git_repository_spec(self, git_dir): """Infer GitRepositorySpec from a local git repository.""" git_text = self.check_run(git_dir, 'remote -v') remote_urls = { match.group(1): match.group(2) for match in re.finditer(r'(\w+)\s+(\S+)\s+\(fetch\)', git_text) } origin_url = remote_urls.get('origin') if not origin_url: raise_and_log_error( UnexpectedError('{0} has no remote "origin"'.format(git_dir))) return GitRepositorySpec(os.path.basename(git_dir), git_dir=git_dir, origin=origin_url, upstream=remote_urls.get('upstream'))
def build_swagger_docs(self, repository, json_path): """Build the API from the swagger endpoint.""" if repository.name != 'gate': raise_and_log_error( UnexpectedError('Repo "%s" != "gate"' % repository.name)) docs_dir = os.path.dirname(json_path) check_subprocess( 'java -jar {jar_path} generate -i {json_path} -l html2' ' -o {output_dir} -t {templates_directory}'.format( jar_path=self.options.swagger_codegen_cli_jar_path, json_path=json_path, output_dir=docs_dir, templates_directory=self.__templates_directory)) logging.info('Writing docs to directory %s', docs_dir)
def ingest_bom(self, line): """Function to ingest a single bom into the result map.""" bom = self.load_bom_from_url(line) if not bom: return try: if bom['version'] + '.yml' != line[line.rfind('/') + 1:]: message = 'BOM version "%s" != filename "%s"' % (bom['version'], line) self.__bad_files[self.url_to_bom_name(line.strip())] = message logging.warning(message) raise_and_log_error(UnexpectedError(message)) self.analyze_bom(bom) except Exception as ex: self.__bad_files[self.url_to_bom_name(line.strip())] = ex.message maybe_log_exception('analyze_bom', ex, action_msg='Skipping %s' % line)
def ensure_git_path(self, repository, **kwargs): """Make sure repository path is consistent with BOM.""" check_kwargs_empty(kwargs) service_name = self.repository_name_to_service_name(repository.name) if not service_name in self.__bom['services'].keys(): raise_and_log_error( UnexpectedError('"%s" is not a BOM repo' % service_name)) git_dir = repository.git_dir have_git_dir = os.path.exists(git_dir) service = check_bom_service(self.__bom, service_name) commit_id = service['commit'] if not have_git_dir: self.git.clone_repository_to_path(repository, commit=commit_id)
def register(self, registry, subparsers, defaults): """Registers a command factory. Args: registry: [dict] The registry to add to, keyed by command name. subparsers: [ArgumentParser subparsers] for adding command arguments defaults: [dict] optional default values for command arguments """ factory = self name = factory.name if name in registry.keys(): raise_and_log_error( UnexpectedError( 'CommandFactory "{name}" already exists.'.format(name=name))) factory.add_argparser(subparsers, defaults) registry[name] = factory
def patchable(self): """Return True if the changes in this repository is only a patch release.""" previous_parts = self.prev_version.split('.') current_parts = self.version.split('.') if len(previous_parts) != 3: raise_and_log_error( ConfigError('Previous version %s is not X.Y.Z' % self.prev_version)) if len(current_parts) != 3: raise_and_log_error( ConfigError('Version %s is not X.Y.Z' % self.version)) if previous_parts[:2] != current_parts[:2]: return False if int(previous_parts[2]) != int(current_parts[2]) - 1: raise_and_log_error( UnexpectedError( 'Unexpected version sequence {prev} to {current}'.format( prev=self.prev_version, current=self.version))) return True
def build_swagger_docs(self, repository, docs_url): """Build the API from the swagger endpoint.""" if repository.name != 'gate': raise_and_log_error( UnexpectedError('Repo "%s" != "gate"' % repository.name)) docs_dir = self.get_output_dir() ensure_dir_exists(docs_dir) docs_path = os.path.join(docs_dir, 'docs.json') logging.info('Generating swagger docs for %s', repository.name) check_subprocess('curl -s {url} -o {docs_path}' .format(url=docs_url, docs_path=docs_path)) check_subprocess( 'java -jar {jar_path} generate -i {docs_path} -l html2' ' -o {output_dir} -t {templates_directory}' .format(jar_path=self.options.swagger_codegen_cli_jar_path, docs_path=docs_path, output_dir=docs_dir, templates_directory=self.__templates_directory)) logging.info('Writing docs to directory %s', docs_dir)
def __determine_repo_install_args(self, repository): """Determine --spinnaker_dev-github_[owner|user] args for install script.""" options = self.options branch = options.git_branch owner = ('spinnaker' if options.github_owner in ('default', 'upstream') else options.github_owner) git_dir = os.path.dirname(__file__) if not branch: branch = GitRunner(options).query_local_repository_branch(git_dir) if not owner: url = repository.origin match = re.search('github.com/([^/]+)/', url) if not match: raise_and_log_error( UnexpectedError('Cannot determine owner from url=%s' % url, cause='BadUrl')) owner = match.group(1) return [ '--spinnaker_dev_github_owner', owner, '--spinnaker_dev_github_branch', branch ]
def _do_repository(self, repository): if repository.name != SPINNAKER_IO_REPOSITORY_NAME: raise_and_log_error(UnexpectedError('Got "%s"' % repository.name)) base_branch = "master" self.scm.ensure_git_path(repository, branch=base_branch) version = self.options.spinnaker_version if self.options.git_allow_publish_master_branch: branch_flag = "" head_branch = "master" else: branch_flag = "-B" head_branch = version + "-changelog" files_added = self.prepare_local_repository_files(repository) git_dir = repository.git_dir message = "doc(changelog): Spinnaker Version " + version local_git_commands = [ # These commands are accomodating to a branch already existing # because the branch is on the version, not build. A rejected # build for some reason that is re-tried will have the same version # so the branch may already exist from the earlier attempt. "fetch origin " + base_branch, "checkout " + base_branch, "checkout {flag} {branch}".format(flag=branch_flag, branch=head_branch), "add " + " ".join([os.path.abspath(path) for path in files_added]), ] logging.debug( 'Commiting changes into local repository "%s" branch=%s', repository.git_dir, head_branch, ) git = self.git git.check_run_sequence(git_dir, local_git_commands) git.check_commit_or_no_changes(git_dir, '-m "{msg}"'.format(msg=message)) logging.info('Pushing branch="%s" into "%s"', head_branch, repository.origin) git.push_branch_to_origin(git_dir, branch=head_branch)
def load_bom(options): """Helper function for initializing the BOM if one was specified.""" bom_path = options.bom_path if hasattr(options, 'bom_path') else None bom_version = (options.bom_version if hasattr(options, 'bom_version') else None) have_bom_path = 1 if bom_path else 0 have_bom_version = 1 if bom_version else 0 if have_bom_path + have_bom_version != 1: raise_and_log_error( ConfigError('Expected exactly one of: "bom_path", or "bom_version"')) if bom_path: check_path_exists(bom_path, why='options.bom_path') return BomSourceCodeManager.bom_from_path(bom_path) if bom_version: logging.debug('Retrieving bom version %s', bom_version) return HalRunner(options).retrieve_bom_version(bom_version) raise_and_log_error(UnexpectedError('Not reachable', cause='NotReachable'))
def make(entry): """Create a new CommitMessage from an individual entry""" match = CommitMessage._MEDIUM_PRETTY_COMMIT_MATCHER.match(entry) if match is None: raise_and_log_error( UnexpectedError('Unexpected commit entry {0}'.format(entry))) text = entry[match.end(3):] # strip trailing spaces on each line lines = [line.rstrip() for line in text.split('\n')] # remove blank lines from beginning and end of text while lines and not lines[0]: del lines[0] while lines and not lines[-1]: del lines[-1] # new string may have initial spacing but no leading/trailing blank lines. text = '\n'.join(lines) return CommitMessage(match.group(1), match.group(2), match.group(3), text)
def __determine_repo_install_args(self, repository): """Determine --spinnaker_dev-github_[owner|user] args for install script.""" options = self.options branch = options.git_branch owner = ("spinnaker" if options.github_owner in ("default", "upstream") else options.github_owner) git_dir = os.path.dirname(__file__) if not branch: branch = GitRunner(options).query_local_repository_branch(git_dir) if not owner: url = repository.origin match = re.search("github.com/([^/]+)/", url) if not match: raise_and_log_error( UnexpectedError("Cannot determine owner from url=%s" % url, cause="BadUrl")) owner = match.group(1) return [ "--spinnaker_dev_github_owner", owner, "--spinnaker_dev_github_branch", branch, ]
def prepare_local_repository_files(self, repository): if repository.name != SPINNAKER_GITHUB_IO_REPOSITORY_NAME: raise_and_log_error(UnexpectedError('Got "%s"' % repository.name)) with open(self.__markdown_path) as f: detail = f.read() # Use the original capture time utc = datetime.datetime.fromtimestamp( os.path.getmtime(self.__markdown_path)) timestamp = '{:%Y-%m-%d %H:%M:%S %Z}'.format(utc) version = self.options.spinnaker_version changelog_filename = '{version}-changelog.md'.format(version=version) target_path = os.path.join(repository.git_dir, '_changelogs', changelog_filename) major, minor, _ = version.split('.') logging.debug('Adding changelog file %s', target_path) with open(target_path, 'w') as f: # pylint: disable=trailing-whitespace header = textwrap.dedent( """\ --- title: Version {version} date: {timestamp} tags: changelogs {major}.{minor} --- # Spinnaker {version} """.format( version=version, timestamp=timestamp, major=major, minor=minor)) f.write(header) f.write(detail) return [target_path]
def determine_origin(self, name): """Determine the origin to use for the given repository.""" if not self.__github_owner: raise_and_log_error( UnexpectedError('Not reachable', cause='NotReachable')) return self.determine_origin_for_owner(name, self.__github_owner)