class ChartBuilder(object): ''' This class handles taking chart intentions as a parameter and turning those into proper protoc helm charts that can be pushed to tiller. It also processes chart source declarations, fetching chart source from external resources where necessary ''' _logger = logger.get_logger('ChartBuilder') def __init__(self, chart, parent=None): ''' Initialize the ChartBuilder class Note that tthis will trigger a source pull as part of initialization as its necessary in order to examine the source service many of the calls on ChartBuilder ''' # cache for generated protoc chart object self._helm_chart = None # record whether this is a dependency based chart self.parent = parent # store chart schema self.chart = dotify(chart) # extract, pull, whatever the chart from its source self.source_directory = self.source_clone() def source_clone(self): ''' Clone the charts source We only support a git source type right now, which can also handle git:// local paths as well ''' subpath = self.chart.source.get('subpath', '') if not 'type' in self.chart.source: self._logger.exception("Need source type for chart %s", self.chart.name) return if self.parent: self._logger.info("Cloning %s/%s as dependency for %s", self.chart.source.location, subpath, self.parent) else: self._logger.info("Cloning %s/%s for release %s", self.chart.source.location, subpath, self.chart.name) if self.chart.source.type == 'git': if 'reference' not in self.chart.source: self.chart.source.reference = 'master' if 'path' not in self.chart.source: self.chart.source.path = '' self._source_tmp_dir = repo.git_clone(self.chart.source.location, self.chart.source.reference, self.chart.source.path) elif self.chart.source.type == 'repo': if 'version' not in self.chart: self.chart.version = None if 'headers' not in self.chart.source: self.chart.source.headers = None self._source_tmp_dir = repo.from_repo(self.chart.source.location, self.chart.name, self.chart.version, self.chart.source.headers) elif self.chart.source.type == 'directory': self._source_tmp_dir = self.chart.source.location else: self._logger.exception("Unknown source type %s for chart %s", self.chart.name, self.chart.source.type) return return os.path.join(self._source_tmp_dir, subpath) def source_cleanup(self): ''' Cleanup source ''' repo.source_cleanup(self._source_tmp_dir) def get_metadata(self): ''' Process metadata ''' # extract Chart.yaml to construct metadata with open(os.path.join(self.source_directory, 'Chart.yaml')) as fd: chart_yaml = dotify(yaml.load(fd.read())) # construct Metadata object return Metadata(description=chart_yaml.description, name=chart_yaml.name, version=chart_yaml.version) def get_files(self): ''' Return (non-template) files in this chart ''' # TODO(yanivoliver): add support for .helmignore # TODO(yanivoliver): refactor seriously to be similar to what Helm does # (https://github.com/helm/helm/blob/master/pkg/chartutil/load.go) chart_files = [] for root, _, files in os.walk(self.source_directory): if root.endswith("charts") or root.endswith("templates"): continue for file in files: if file in (".helmignore", "Chart.yaml", "values.toml", "values.yaml"): continue filename = os.path.relpath(os.path.join(root, file), self.source_directory) # TODO(yanivoliver): Find a better solution. # We need this in order to support charts on Windows - Tiller will look # for the files it uses using the relative path, using Linuxish # path seperators (/). Thus, sending the file list to Tiller # from a Windows machine the lookup will fail. filename = filename.replace("\\", "/") with open(os.path.join(root, file), "r") as fd: chart_files.append(Any(type_url=filename, value=fd.read())) return chart_files def get_values(self): ''' Return the chart (default) values ''' # create config object representing unmarshaled values.yaml if os.path.exists(os.path.join(self.source_directory, 'values.yaml')): with open(os.path.join(self.source_directory, 'values.yaml')) as fd: raw_values = fd.read() else: self._logger.warn("No values.yaml in %s, using empty values", self.source_directory) raw_values = '' return Config(raw=raw_values) def get_templates(self): ''' Return all the chart templates ''' # process all files in templates/ as a template to attach to the chart # building a Template object templates = [] if not os.path.exists(os.path.join(self.source_directory, 'templates')): self._logger.warn( "Chart %s has no templates directory, " "no templates will be deployed", self.chart.name) for root, _, files in os.walk( os.path.join(self.source_directory, 'templates')): for tpl_file in files: template_name = os.path.relpath(os.path.join(root, tpl_file), self.source_directory) # TODO(yanivoliver): Find a better solution. # We need this in order to support charts on Windows - Tiller will look # for the templates it uses using the relative path, using Linuxish # path seperators (/). Thus, sending the template list to Tiller # from a Windows machine the lookup will fail. template_name = template_name.replace("\\", "/") templates.append( Template(name=template_name, data=open(os.path.join(root, tpl_file), 'r').read())) return templates def get_helm_chart(self): ''' Return a helm chart object ''' if self._helm_chart: return self._helm_chart dependencies = [] for chart in self.chart.get('dependencies', []): self._logger.info("Building dependency chart %s for release %s", chart.name, self.chart.name) dependencies.append(ChartBuilder(chart).get_helm_chart()) helm_chart = Chart( metadata=self.get_metadata(), templates=self.get_templates(), dependencies=dependencies, values=self.get_values(), files=self.get_files(), ) self._helm_chart = helm_chart return helm_chart def dump(self): ''' This method is used to dump a chart object as a serialized string so that we can perform a diff It should recurse into dependencies ''' return self.get_helm_chart().SerializeToString()
class ChartBuilder(object): """ This class handles taking chart intentions as a parameter and turning those into proper protoc helm charts that can be pushed to tiller. It also processes chart source declarations, fetching chart source from external resources where necessary """ _logger = logger.get_logger("ChartBuilder") def __init__(self, chart, parent=None): """ Initialize the ChartBuilder class Note that tthis will trigger a source pull as part of initialization as its necessary in order to examine the source service many of the calls on ChartBuilder """ # cache for generated protoc chart object self._helm_chart = None # record whether this is a dependency based chart self.parent = parent # store chart schema self.chart = dotify(chart) # extract, pull, whatever the chart from its source self.source_directory = self.source_clone() def source_clone(self): """ Clone the charts source We only support a git source type right now, which can also handle git:// local paths as well """ subpath = self.chart.source.get("subpath", "") if "name" not in self.chart: self._logger.exception("Please specify name for the chart") return if "type" not in self.chart.source: self._logger.exception("Need source type for chart %s", self.chart.name) return if self.parent: self._logger.info( "Cloning %s/%s as dependency for %s", self.chart.source.location, subpath, self.parent, ) else: self._logger.info( "Cloning %s/%s for release %s", self.chart.source.location, subpath, self.chart.name, ) if self.chart.source.type == "git": if "reference" not in self.chart.source: self.chart.source.reference = "master" if "path" not in self.chart.source: self.chart.source.path = "" self._source_tmp_dir = repo.git_clone( self.chart.source.location, self.chart.source.reference, self.chart.source.path, ) elif self.chart.source.type == "repo": if "version" not in self.chart: self.chart.version = None if "headers" not in self.chart.source: self.chart.source.headers = None self._source_tmp_dir = repo.from_repo( self.chart.source.location, self.chart.name, self.chart.version, self.chart.source.headers, ) elif self.chart.source.type == "directory": self._source_tmp_dir = self.chart.source.location else: self._logger.exception( "Unknown source type %s for chart %s", self.chart.name, self.chart.source.type, ) return return os.path.join(self._source_tmp_dir, subpath) def source_cleanup(self): """ Cleanup source """ repo.source_cleanup(self._source_tmp_dir) def get_metadata(self): """ Process metadata """ # extract Chart.yaml to construct metadata chart_yaml = yaml.load( ChartBuilder.read_file( os.path.join(self.source_directory, "Chart.yaml"))) if "version" not in chart_yaml or "name" not in chart_yaml: self._logger.error("Chart missing required fields") return default_chart_yaml = defaultdict(str, chart_yaml) # construct Metadata object return Metadata( apiVersion=default_chart_yaml["apiVersion"], description=default_chart_yaml["description"], name=default_chart_yaml["name"], version=str(default_chart_yaml["version"]), appVersion=str(default_chart_yaml["appVersion"]), ) def get_files(self): """ Return (non-template) files in this chart """ # TODO(yanivoliver): add support for .helmignore # TODO(yanivoliver): refactor seriously to be similar to what Helm does # (https://github.com/helm/helm/blob/master/pkg/chartutil/load.go) chart_files = [] for root, _, files in os.walk(self.source_directory): if root.endswith("charts") or root.endswith("templates"): continue for file in files: if file in (".helmignore", "Chart.yaml", "values.toml", "values.yaml"): continue filename = os.path.relpath(os.path.join(root, file), self.source_directory) # TODO(yanivoliver): Find a better solution. # We need this in order to support charts on Windows - Tiller will look # for the files it uses using the relative path, using Linuxish # path seperators (/). Thus, sending the file list to Tiller # from a Windows machine the lookup will fail. filename = filename.replace("\\", "/") chart_files.append( Any( type_url=filename, value=ChartBuilder.read_file(os.path.join(root, file)), )) return chart_files def get_values(self): """ Return the chart (default) values """ # create config object representing unmarshaled values.yaml if os.path.exists(os.path.join(self.source_directory, "values.yaml")): raw_values = ChartBuilder.read_file( os.path.join(self.source_directory, "values.yaml")) else: self._logger.warn("No values.yaml in %s, using empty values", self.source_directory) raw_values = "" return Config(raw=raw_values) def get_templates(self): """ Return all the chart templates """ # process all files in templates/ as a template to attach to the chart # building a Template object templates = [] if not os.path.exists(os.path.join(self.source_directory, "templates")): self._logger.warn( "Chart %s has no templates directory, " "no templates will be deployed", self.chart.name, ) for root, _, files in os.walk( os.path.join(self.source_directory, "templates")): for tpl_file in files: template_name = os.path.relpath(os.path.join(root, tpl_file), self.source_directory) # TODO(yanivoliver): Find a better solution. # We need this in order to support charts on Windows - Tiller will look # for the templates it uses using the relative path, using Linuxish # path seperators (/). Thus, sending the template list to Tiller # from a Windows machine the lookup will fail. template_name = template_name.replace("\\", "/") templates.append( Template( name=template_name, data=ChartBuilder.read_file( os.path.join(root, tpl_file)), )) return templates def get_helm_chart(self): """ Return a helm chart object """ if self._helm_chart: return self._helm_chart dependencies = [] for chart in self.chart.get("dependencies", []): self._logger.info( "Building dependency chart %s for release %s", chart.name, self.chart.name, ) dependencies.append(ChartBuilder(chart).get_helm_chart()) helm_chart = Chart( metadata=self.get_metadata(), templates=self.get_templates(), dependencies=dependencies, values=self.get_values(), files=self.get_files(), ) self._helm_chart = helm_chart return helm_chart @staticmethod def read_file(path): """ Open the file provided in `path` and strip any non-UTF8 characters. Return back the cleaned content """ with codecs.open(path, encoding="utf-8", errors="ignore") as fd: content = fd.read() return bytes(bytearray(content, encoding="utf-8")) def dump(self): """ This method is used to dump a chart object as a serialized string so that we can perform a diff It should recurse into dependencies """ return self.get_helm_chart().SerializeToString()
class Tiller(object): """ The Tiller class supports communication and requests to the Tiller Helm service over gRPC """ _logger = logger.get_logger('Tiller') def __init__(self, host, port=TILLER_PORT, timeout=TILLER_TIMEOUT, tls_config=None): # init k8s connectivity self._host = host self._port = port self._tls_config = tls_config # init tiller channel self._channel = self.get_channel() # init timeout for all requests self._timeout = timeout @property def metadata(self): """ Return tiller metadata for requests """ return [(b'x-helm-api-client', TILLER_VERSION)] def get_channel(self): """ Return a tiller channel """ target = '%s:%s' % (self._host, self._port) if self._tls_config: ssl_channel_credentials = grpc.ssl_channel_credentials( root_certificates=self._tls_config.ca_data, private_key=self._tls_config.key_data, certificate_chain=self._tls_config.cert_data) return grpc.secure_channel(target, ssl_channel_credentials) else: return grpc.insecure_channel(target) def tiller_status(self): """ return if tiller exist or not """ if self._host: return True return False def list_releases(self, status_codes=None, namespace=""): """ List Helm Releases Possible status codes can be seen in the status_pb2 in part of Helm gRPC definition """ releases = [] # Convert the string status codes to the their numerical values if status_codes: codes_enum = _STATUS.enum_types_by_name.get("Code") request_status_codes = [ codes_enum.values_by_name.get(code).number for code in status_codes ] else: request_status_codes = [] offset = None stub = ReleaseServiceStub(self._channel) while True: req = ListReleasesRequest(limit=RELEASE_LIMIT, offset=offset, namespace=namespace, status_codes=request_status_codes) release_list = stub.ListReleases(req, self._timeout, metadata=self.metadata) for y in release_list: offset = str(y.next) releases.extend(y.releases) # This handles two cases: # 1. If there are no releases, offset will not be set and will remain None # 2. If there were releases, once we've fetched all of them, offset will be "" if not offset: break return releases def list_charts(self): """ List Helm Charts from Latest Releases Returns list of (name, version, chart, values) """ charts = [] for latest_release in self.list_releases(): try: charts.append( (latest_release.name, latest_release.version, latest_release.chart, latest_release.config.raw)) except IndexError: continue return charts def update_release(self, chart, namespace, dry_run=False, name=None, values=None, wait=False, disable_hooks=False, recreate=False, reset_values=False, reuse_values=False, force=False, description="", install=False): """ Update a Helm Release """ stub = ReleaseServiceStub(self._channel) if install: if not namespace: namespace = DEFAULT_NAMESPACE try: release_status = self.get_release_status(name) except grpc.RpcError as rpc_error_call: if not rpc_error_call.details( ) == "getting deployed release \"{}\": release: \"{}\" not found".format( name, name): raise rpc_error_call # The release doesn't exist - it's time to install self._logger.info( "Release %s does not exist. Installing it now.", name) return self.install_release(chart, namespace, dry_run, name, values, wait) if release_status.namespace != namespace: self._logger.warn( "Namespace %s doesn't match with previous. Release will be deployed to %s", release_status.namespace, namespace) values = Config(raw=yaml.safe_dump(values or {})) release_request = UpdateReleaseRequest(chart=chart, dry_run=dry_run, disable_hooks=disable_hooks, values=values, name=name or '', wait=wait, recreate=recreate, reset_values=reset_values, reuse_values=reuse_values, force=force, description=description) return stub.UpdateRelease(release_request, self._timeout, metadata=self.metadata) def install_release(self, chart, namespace, dry_run=False, name=None, values=None, wait=False, disable_hooks=False, reuse_name=False, disable_crd_hook=False, description=""): """ Create a Helm Release """ values = Config(raw=yaml.safe_dump(values or {})) stub = ReleaseServiceStub(self._channel) release_request = InstallReleaseRequest( chart=chart, dry_run=dry_run, values=values, name=name or '', namespace=namespace, wait=wait, disable_hooks=disable_hooks, reuse_name=reuse_name, disable_crd_hook=disable_crd_hook, description=description) return stub.InstallRelease(release_request, self._timeout, metadata=self.metadata) def uninstall_release(self, release, disable_hooks=False, purge=True): """ :params - release - helm chart release name :params - purge - deep delete of chart Deletes a helm chart from tiller """ stub = ReleaseServiceStub(self._channel) release_request = UninstallReleaseRequest(name=release, disable_hooks=disable_hooks, purge=purge) return stub.UninstallRelease(release_request, self._timeout, metadata=self.metadata) def get_release_status(self, release, version=None): """ Gets a release's status """ stub = ReleaseServiceStub(self._channel) status_request = GetReleaseStatusRequest(name=release, version=version) return stub.GetReleaseStatus(status_request, self._timeout, metadata=self.metadata) def get_release_content(self, release, version=None): """ Gets a release's content """ stub = ReleaseServiceStub(self._channel) status_request = GetReleaseContentRequest(name=release, version=version) return stub.GetReleaseContent(status_request, self._timeout, metadata=self.metadata) def chart_cleanup(self, prefix, charts): """ :params charts - list of yaml charts :params known_release - list of releases in tiller :result - will remove any chart that is not present in yaml """ def release_prefix(prefix, chart): """ how to attach prefix to chart """ return "{}-{}".format(prefix, chart["chart"]["release_name"]) valid_charts = [release_prefix(prefix, chart) for chart in charts] actual_charts = [x.name for x in self.list_releases()] chart_diff = list(set(actual_charts) - set(valid_charts)) for chart in chart_diff: if chart.startswith(prefix): self._logger.debug("Release: %s will be removed", chart) self.uninstall_release(chart)
class ChartBuilder(object): ''' This class handles taking chart intentions as a parameter and turning those into proper protoc helm charts that can be pushed to tiller. It also processes chart source declarations, fetching chart source from external resources where necessary ''' _logger = logger.get_logger('ChartBuilder') def __init__(self, chart, values_files, parent=None): ''' Initialize the ChartBuilder class Note that tthis will trigger a source pull as part of initialization as its necessary in order to examine the source service many of the calls on ChartBuilder ''' # cache for generated protoc chart object self._helm_chart = None # record whether this is a dependency based chart self.parent = parent # store chart schema self.chart = dotify(chart) # extract, pull, whatever the chart from its source self.source_directory = self.source_clone() # n.b.: these are later referred to as `overrides` # but they are what replace values in the chart/values.yaml # when handled by `Tiller` self.values_files = values_files def source_clone(self): ''' Clone the charts source We only support a git source type right now, which can also handle git:// local paths as well ''' subpath = self.chart.source.get('subpath', '') if not 'type' in self.chart.source: self._logger.exception("Need source type for chart %s", self.chart.name) return if self.parent: self._logger.info("Cloning %s/%s as dependency for %s", self.chart.source.location, subpath, self.parent) else: self._logger.info("Cloning %s/%s for release %s", self.chart.source.location, subpath, self.chart.name) if self.chart.source.type == 'git': if 'reference' not in self.chart.source: self.chart.source.reference = 'master' if 'path' not in self.chart.source: self.chart.source.path = '' self._source_tmp_dir = repo.git_clone(self.chart.source.location, self.chart.source.reference, self.chart.source.path) elif self.chart.source.type == 'repo': if 'version' not in self.chart: self.chart.version = None if 'headers' not in self.chart.source: self.chart.source.headers = None self._source_tmp_dir = repo.from_repo(self.chart.source.location, self.chart.name, self.chart.version, self.chart.source.headers) elif self.chart.source.type == 'directory': self._source_tmp_dir = self.chart.source.location else: self._logger.exception("Unknown source type %s for chart %s", self.chart.name, self.chart.source.type) return return os.path.join(self._source_tmp_dir, subpath) def source_cleanup(self): ''' Cleanup source ''' repo.source_cleanup(self._source_tmp_dir) def get_metadata(self): ''' Process metadata ''' # extract Chart.yaml to construct metadata chart_yaml = yaml.safe_load( ChartBuilder.read_file( os.path.join(self.source_directory, 'Chart.yaml'))) if 'version' not in chart_yaml or \ 'name' not in chart_yaml: self._logger.error("Chart missing required fields") return default_chart_yaml = defaultdict(str, chart_yaml) # construct Metadata object return Metadata(apiVersion=default_chart_yaml['apiVersion'], description=default_chart_yaml['description'], name=default_chart_yaml['name'], version=str(default_chart_yaml['version']), appVersion=str(default_chart_yaml['appVersion'])) @staticmethod def is_ignorable(root): if not root.endswith("charts") and not root.endswith("templates"): hidden_files = [ str_split for str_split in root.split(os.sep)[1:] if str_split.startswith('.') ] return len(hidden_files) > 0 return True def get_files(self): ''' Return (non-template) files in this chart ''' # TODO(yanivoliver): refactor seriously to be similar to what Helm does # (https://github.com/helm/helm/blob/master/pkg/chartutil/load.go) chart_files = [] for root, _, files in os.walk(self.source_directory): if not ChartBuilder.is_ignorable(root): helmignore_list = ChartBuilder.get_helmignore(root=root) absolute_paths = [os.path.join(root, file) for file in files] yaml_files = ChartBuilder.remove_helmignored_files( files=absolute_paths, helmignore_list=helmignore_list) yaml_files = ChartBuilder.remove_necessary_files( yaml_files=yaml_files) for file in yaml_files: filename = os.path.relpath(file, self.source_directory) # TODO(yanivoliver): Find a better solution. # We need this in order to support charts on Windows - Tiller will look # for the files it uses using the relative path, using Linuxish # path seperators (/). Thus, sending the file list to Tiller # from a Windows machine the lookup will fail. filename = filename.replace("\\", "/") chart_files.append( Any(type_url=filename, value=ChartBuilder.read_file(file))) return chart_files @staticmethod def remove_necessary_files(yaml_files): yaml_files = [ file for file in yaml_files if file.endswith('.yaml') and not file.endswith("Chart.yaml") and not file.endswith("values.yaml") ] return yaml_files @staticmethod def remove_helmignored_files(files, helmignore_list): helmignored = [file for file in files if file not in helmignore_list] return helmignored @staticmethod def get_helmignore(root): if os.path.exists(f"{root}/.helmignore"): with open(f"{root}/.helmignore", 'r+') as helmignore_file: helmignore_files = [ os.path.join(root, line.rstrip('\n')) for line in helmignore_file.readlines() if not line.lstrip().startswith('#') ] helmignore_file.close() return helmignore_files return [] def get_overrides(self): ''' Return the files intended to override the chart values/values.yaml entries :return: overrides_list ''' return [ file for file in self.get_files() if file.type_url in self.values_files ] def get_values(self): ''' Return the chart (default) values ''' # create config object representing unmarshaled values.yaml if os.path.exists(os.path.join(self.source_directory, 'values.yaml')): raw_values = ChartBuilder.read_file( os.path.join(self.source_directory, 'values.yaml')) else: self._logger.warn("No values.yaml in %s, using empty values", self.source_directory) raw_values = '' return Config(raw=raw_values) def get_templates(self): ''' Return all the chart templates ''' # process all files in templates/ as a template to attach to the chart # building a Template object templates = [] if not os.path.exists(os.path.join(self.source_directory, 'templates')): self._logger.warn( "Chart %s has no templates directory, " "no templates will be deployed", self.chart.name) for root, _, files in os.walk( os.path.join(self.source_directory, 'templates')): for tpl_file in files: template_name = os.path.relpath(os.path.join(root, tpl_file), self.source_directory) # TODO(yanivoliver): Find a better solution. # We need this in order to support charts on Windows - Tiller will look # for the templates it uses using the relative path, using Linuxish # path seperators (/). Thus, sending the template list to Tiller # from a Windows machine the lookup will fail. template_name = template_name.replace("\\", "/") templates.append( Template(name=template_name, data=ChartBuilder.read_file( os.path.join(root, tpl_file)))) return templates def get_helm_chart(self): ''' Return a helm chart object ''' if self._helm_chart: return self._helm_chart dependencies = [] for chart in self.chart.get('dependencies', []): self._logger.info("Building dependency chart %s for release %s", chart.name, self.chart.name) dependencies.append(ChartBuilder(chart).get_helm_chart()) helm_chart = Chart(metadata=self.get_metadata(), templates=self.get_templates(), dependencies=dependencies, values=self.get_values(), files=self.get_files()) self._helm_chart = helm_chart return helm_chart @staticmethod def read_file(path): ''' Open the file provided in `path` and strip any non-UTF8 characters. Return back the cleaned content ''' with codecs.open(path, encoding='utf-8', errors='ignore') as fd: content = fd.read() return bytes(bytearray(content, encoding='utf-8')) def dump(self): ''' This method is used to dump a chart object as a serialized string so that we can perform a diff It should recurse into dependencies ''' return self.get_helm_chart().SerializeToString()