def process_clowd_env(target_ns, quay_user, env_name, template_path): log.info("processing ClowdEnvironment") env_template_path = Path( template_path if template_path else conf.DEFAULT_CLOWDENV_TEMPLATE) if not env_template_path.exists(): raise ValueError("ClowdEnvironment template file does not exist: %s", env_template_path) with env_template_path.open() as fp: template_data = yaml.safe_load(fp) params = dict() params["ENV_NAME"] = env_name if quay_user: quay_user = quay_user.replace("_", "-") params["PULL_SECRET_NAME"] = f"{quay_user}-pull-secret" if target_ns: params["NAMESPACE"] = target_ns processed_template = process_template(template_data, params=params) if not processed_template.get("items"): raise ValueError("Processed ClowdEnvironment template has no items") return processed_template
def process_iqe_cji( clowd_app_name, debug=False, marker="", filter="", env="clowder_smoke", image_tag="", cji_name=None, template_path=None, requirements="", requirements_priority="", test_importance="", ): log.info("processing IQE ClowdJobInvocation") template_path = Path(template_path if template_path else conf.DEFAULT_IQE_CJI_TEMPLATE) if not template_path.exists(): raise FatalError("CJI template file does not exist: %s", template_path) with template_path.open() as fp: template_data = yaml.safe_load(fp) requirements = requirements.split(",") if requirements else [] requirements_priority = requirements_priority.split(",") if requirements_priority else [] test_importance = test_importance.split(",") if test_importance else [] params = dict() params["DEBUG"] = str(debug).lower() params["MARKER"] = marker params["FILTER"] = filter params["ENV_NAME"] = env params["IMAGE_TAG"] = image_tag params["NAME"] = cji_name or f"iqe-{str(uuid.uuid4()).split('-')[0]}" params["APP_NAME"] = clowd_app_name params["REQUIREMENTS"] = json.dumps(requirements) params["REQUIREMENTS_PRIORITY"] = json.dumps(requirements_priority) params["TEST_IMPORTANCE"] = json.dumps(test_importance) processed_template = process_template(template_data, params=params) if not processed_template.get("items"): raise FatalError("Processed CJI template has no items") return processed_template
def add_base_resources(namespace, secret_names): copy_namespace_secrets(conf.BASE_NAMESPACE_NAME, namespace, secret_names) with open(conf.EPHEMERAL_CLUSTER_CLOWDENV_TEMPLATE) as fp: template_data = yaml.safe_load(fp) processed_template = process_template( template_data, params={ "ENV_NAME": conf.ENV_NAME_FORMAT.format(namespace=namespace), "NAMESPACE": namespace, }, ) oc("apply", f="-", _in=json.dumps(processed_template)) # wait for any deployed base resources to become 'ready' wait_for_all_resources(namespace, timeout=conf.RECONCILE_TIMEOUT)
def _process_app(namespace, app_name, apps_cfg, config, k8s_list, get_dependencies, processed_apps): app_cfg = apps_cfg[app_name] if app_cfg["host"] == "gitlab": commit, template_content = process_gitlab(app_cfg) elif app_cfg["host"] == "github": commit, template_content = process_github(app_cfg) elif app_cfg["host"] == "local": commit, template_content = process_local(app_cfg) else: raise ValueError("invalid host %s for app %s" % (app_cfg["host"], app_cfg["name"])) template = yaml.safe_load(template_content) params = { "IMAGE_TAG": commit[:7], "ENV_NAME": config.get("envName") or conf.ENV_NAME_FORMAT.format(namespace=namespace), "CLOWDER_ENABLED": "true", "MIN_REPLICAS": "1", "REPLICAS": "1", } params.update(app_cfg.get("parameters", {})) new_items = process_template(template, params)["items"] _remove_resource_config(new_items) k8s_list["items"].extend(new_items) processed_apps.add(app_name) if get_dependencies: items = _add_dependencies_to_config(namespace, app_name, new_items, processed_apps, config) k8s_list["items"].extend(items)
def _get_component_items(self, component_name): component = self._get_component_config(component_name) try: rf = RepoFile.from_config(component) # override template ref if requested self._sub_ref(component_name, rf) commit, template_content = rf.fetch() except Exception as err: log.error("failed to fetch template file for %s", component_name) log.debug(traceback.format_exc()) raise FatalError(err) template = yaml.safe_load(template_content) params = { "IMAGE_TAG": commit[:7], "ENV_NAME": self.clowd_env, } params.update(component.get("parameters", {})) # override any specific parameters on this component if requested self._sub_params(component_name, params) new_items = process_template(template, params)["items"] # override the tags for all occurences of an image if requested new_items = self._sub_image_tags(new_items) if ( "all" not in self.no_remove_resources and ("all" in self.remove_resources or component_name in self.remove_resources) and component_name not in self.no_remove_resources ): _remove_resource_config(new_items) if self.single_replicas: _set_replicas(new_items) return new_items
def process_iqe_cji( clowd_app_name, debug=False, marker="", filter="", env="clowder_smoke", image_tag="", cji_name=None, template_path=None, ): log.info("processing IQE ClowdJobInvocation") template_path = Path( template_path if template_path else conf.DEFAULT_IQE_CJI_TEMPLATE) if not template_path.exists(): raise ValueError("CJI template file does not exist: %s", template_path) with template_path.open() as fp: template_data = yaml.safe_load(fp) params = dict() params["DEBUG"] = str(debug).lower() params["MARKER"] = marker params["FILTER"] = filter params["ENV_NAME"] = env params["IMAGE_TAG"] = image_tag params["NAME"] = cji_name or f"iqe-{str(uuid.uuid4()).split('-')[0]}" params["APP_NAME"] = clowd_app_name processed_template = process_template(template_data, params=params) if not processed_template.get("items"): raise ValueError("Processed CJI template has no items") return processed_template
def process_reservation( name, requester, duration, template_path=None, ): log.info("processing namespace reservation") template_path = Path(template_path if template_path else conf.DEFAULT_RESERVATION_TEMPLATE) if not template_path.exists(): raise FatalError("Reservation template file does not exist: %s", template_path) with template_path.open() as fp: template_data = yaml.safe_load(fp) params = dict() params["NAME"] = name if name else f"bonfire-reservation-{str(uuid.uuid4()).split('-')[0]}" params["DURATION"] = duration if requester is None: try: requester = whoami() except Exception: log.info("whoami returned an error - setting requester to 'bonfire'") # minikube requester = "bonfire" params["REQUESTER"] = requester processed_template = process_template(template_data, params=params) if not processed_template.get("items"): raise FatalError("Processed Reservation template has no items") return processed_template
def _get_processed_items( client, app, saas_file, src_env, ref_env, src_env_data, ref_env_data, template_ref_overrides, namespace, ): src_resources = _get_resources_for_env(saas_file, src_env_data) log.debug( "found resources for %s in saas file '%s': %s", src_env, saas_file["name"], json.dumps(src_resources, indent=2), ) ref_resources = _get_resources_for_env(saas_file, ref_env_data) log.debug( "found resources for %s in saas file '%s': %s", ref_env, saas_file["name"], json.dumps(ref_resources, indent=2), ) items = [] for resource_name, resource in src_resources.items(): src_targets = resource.get("targets", []) ref_targets = ref_resources.get(resource_name, {}).get("targets", []) log.debug("%s -- parsing targets", _format_app_resource(app, resource_name, saas_file)) src_target, ref_target = _parse_targets( src_targets, ref_targets, app, resource_name, src_env, ref_env, saas_file ) if not src_target: # this resource was not marked for deployment log.warn( "%s -- not marked for deploy in src env, skipping resource!", _format_app_resource(app, resource_name, saas_file), ) continue if resource_name in template_ref_overrides: # if template ref has explicitly been overridden, use the override log.debug( "%s -- overriding template ref", _format_app_resource(app, resource_name, saas_file), ) template_ref = template_ref_overrides[resource_name] elif not ref_target: # if template ref not overridden, and there's no ref target, we don't know what git ref # to use for template download log.warn( "%s -- no ref target found nor template ref override given, defaulting to 'master'", _format_app_resource(app, resource_name, saas_file), ) template_ref = "master" else: # otherwise use template ref configured in the "reference deploy target" template_ref = ref_target["ref"] raw_template = _download_raw_template(resource, template_ref) # merge the various layers of parameters to pass into the template p = copy.deepcopy(json.loads(src_env_data["parameters"])) p.update(saas_file["parameters"]) p.update(resource["parameters"]) p.update(src_target["parameters"]) # set IMAGE_TAG to be the reference env's IMAGE_TAG p["IMAGE_TAG"] = ref_target.get("parameters", {}).get("IMAGE_TAG") if not p.get("IMAGE_TAG"): if template_ref == "master" or template_ref == "main": image_tag = "latest" else: image_tag = template_ref[:7] p.update({"IMAGE_TAG": image_tag}) log.debug( "%s -- no IMAGE_TAG found on ref target, assuming tag '%s' based on template ref", _format_app_resource(app, resource_name, saas_file), p["IMAGE_TAG"], ) # set the env name based on the namespace you intend to deploy to p["ENV_NAME"] = conf.ENV_NAME_FORMAT.format(namespace=namespace) processed_template = process_template(raw_template, p) items.extend(processed_template.get("items", [])) return items