def process_clowd_env(target_ns, quay_user, env_name, template_path):
log.info("processing ClowdEnvironment")
env_template_path = Path(
template_path if template_path else conf.DEFAULT_CLOWDENV_TEMPLATE)
if not env_template_path.exists():
raise ValueError("ClowdEnvironment template file does not exist: %s",
env_template_path)
with env_template_path.open() as fp:
template_data = yaml.safe_load(fp)
params = dict()
params["ENV_NAME"] = env_name
if quay_user:
quay_user = quay_user.replace("_", "-")
params["PULL_SECRET_NAME"] = f"{quay_user}-pull-secret"
if target_ns:
params["NAMESPACE"] = target_ns
processed_template = process_template(template_data, params=params)
if not processed_template.get("items"):
raise ValueError("Processed ClowdEnvironment template has no items")
return processed_template
def process_iqe_cji(
clowd_app_name,
debug=False,
marker="",
filter="",
env="clowder_smoke",
image_tag="",
cji_name=None,
template_path=None,
requirements="",
requirements_priority="",
test_importance="",
):
log.info("processing IQE ClowdJobInvocation")
template_path = Path(template_path if template_path else conf.DEFAULT_IQE_CJI_TEMPLATE)
if not template_path.exists():
raise FatalError("CJI template file does not exist: %s", template_path)
with template_path.open() as fp:
template_data = yaml.safe_load(fp)
requirements = requirements.split(",") if requirements else []
requirements_priority = requirements_priority.split(",") if requirements_priority else []
test_importance = test_importance.split(",") if test_importance else []
params = dict()
params["DEBUG"] = str(debug).lower()
params["MARKER"] = marker
params["FILTER"] = filter
params["ENV_NAME"] = env
params["IMAGE_TAG"] = image_tag
params["NAME"] = cji_name or f"iqe-{str(uuid.uuid4()).split('-')[0]}"
params["APP_NAME"] = clowd_app_name
params["REQUIREMENTS"] = json.dumps(requirements)
params["REQUIREMENTS_PRIORITY"] = json.dumps(requirements_priority)
params["TEST_IMPORTANCE"] = json.dumps(test_importance)
processed_template = process_template(template_data, params=params)
if not processed_template.get("items"):
raise FatalError("Processed CJI template has no items")
return processed_template
def add_base_resources(namespace, secret_names):
copy_namespace_secrets(conf.BASE_NAMESPACE_NAME, namespace, secret_names)
with open(conf.EPHEMERAL_CLUSTER_CLOWDENV_TEMPLATE) as fp:
template_data = yaml.safe_load(fp)
processed_template = process_template(
template_data,
params={
"ENV_NAME": conf.ENV_NAME_FORMAT.format(namespace=namespace),
"NAMESPACE": namespace,
},
)
oc("apply", f="-", _in=json.dumps(processed_template))
# wait for any deployed base resources to become 'ready'
wait_for_all_resources(namespace, timeout=conf.RECONCILE_TIMEOUT)
def _process_app(namespace, app_name, apps_cfg, config, k8s_list,
get_dependencies, processed_apps):
app_cfg = apps_cfg[app_name]
if app_cfg["host"] == "gitlab":
commit, template_content = process_gitlab(app_cfg)
elif app_cfg["host"] == "github":
commit, template_content = process_github(app_cfg)
elif app_cfg["host"] == "local":
commit, template_content = process_local(app_cfg)
else:
raise ValueError("invalid host %s for app %s" %
(app_cfg["host"], app_cfg["name"]))
template = yaml.safe_load(template_content)
params = {
"IMAGE_TAG":
commit[:7],
"ENV_NAME":
config.get("envName")
or conf.ENV_NAME_FORMAT.format(namespace=namespace),
"CLOWDER_ENABLED":
"true",
"MIN_REPLICAS":
"1",
"REPLICAS":
"1",
}
params.update(app_cfg.get("parameters", {}))
new_items = process_template(template, params)["items"]
_remove_resource_config(new_items)
k8s_list["items"].extend(new_items)
processed_apps.add(app_name)
if get_dependencies:
items = _add_dependencies_to_config(namespace, app_name, new_items,
processed_apps, config)
k8s_list["items"].extend(items)
def _get_component_items(self, component_name):
component = self._get_component_config(component_name)
try:
rf = RepoFile.from_config(component)
# override template ref if requested
self._sub_ref(component_name, rf)
commit, template_content = rf.fetch()
except Exception as err:
log.error("failed to fetch template file for %s", component_name)
log.debug(traceback.format_exc())
raise FatalError(err)
template = yaml.safe_load(template_content)
params = {
"IMAGE_TAG": commit[:7],
"ENV_NAME": self.clowd_env,
}
params.update(component.get("parameters", {}))
# override any specific parameters on this component if requested
self._sub_params(component_name, params)
new_items = process_template(template, params)["items"]
# override the tags for all occurences of an image if requested
new_items = self._sub_image_tags(new_items)
if (
"all" not in self.no_remove_resources
and ("all" in self.remove_resources or component_name in self.remove_resources)
and component_name not in self.no_remove_resources
):
_remove_resource_config(new_items)
if self.single_replicas:
_set_replicas(new_items)
return new_items
def process_iqe_cji(
clowd_app_name,
debug=False,
marker="",
filter="",
env="clowder_smoke",
image_tag="",
cji_name=None,
template_path=None,
):
log.info("processing IQE ClowdJobInvocation")
template_path = Path(
template_path if template_path else conf.DEFAULT_IQE_CJI_TEMPLATE)
if not template_path.exists():
raise ValueError("CJI template file does not exist: %s", template_path)
with template_path.open() as fp:
template_data = yaml.safe_load(fp)
params = dict()
params["DEBUG"] = str(debug).lower()
params["MARKER"] = marker
params["FILTER"] = filter
params["ENV_NAME"] = env
params["IMAGE_TAG"] = image_tag
params["NAME"] = cji_name or f"iqe-{str(uuid.uuid4()).split('-')[0]}"
params["APP_NAME"] = clowd_app_name
processed_template = process_template(template_data, params=params)
if not processed_template.get("items"):
raise ValueError("Processed CJI template has no items")
return processed_template
def process_reservation(
name,
requester,
duration,
template_path=None,
):
log.info("processing namespace reservation")
template_path = Path(template_path if template_path else conf.DEFAULT_RESERVATION_TEMPLATE)
if not template_path.exists():
raise FatalError("Reservation template file does not exist: %s", template_path)
with template_path.open() as fp:
template_data = yaml.safe_load(fp)
params = dict()
params["NAME"] = name if name else f"bonfire-reservation-{str(uuid.uuid4()).split('-')[0]}"
params["DURATION"] = duration
if requester is None:
try:
requester = whoami()
except Exception:
log.info("whoami returned an error - setting requester to 'bonfire'") # minikube
requester = "bonfire"
params["REQUESTER"] = requester
processed_template = process_template(template_data, params=params)
if not processed_template.get("items"):
raise FatalError("Processed Reservation template has no items")
return processed_template
def _get_processed_items(
client,
app,
saas_file,
src_env,
ref_env,
src_env_data,
ref_env_data,
template_ref_overrides,
namespace,
):
src_resources = _get_resources_for_env(saas_file, src_env_data)
log.debug(
"found resources for %s in saas file '%s': %s",
src_env,
saas_file["name"],
json.dumps(src_resources, indent=2),
)
ref_resources = _get_resources_for_env(saas_file, ref_env_data)
log.debug(
"found resources for %s in saas file '%s': %s",
ref_env,
saas_file["name"],
json.dumps(ref_resources, indent=2),
)
items = []
for resource_name, resource in src_resources.items():
src_targets = resource.get("targets", [])
ref_targets = ref_resources.get(resource_name, {}).get("targets", [])
log.debug("%s -- parsing targets", _format_app_resource(app, resource_name, saas_file))
src_target, ref_target = _parse_targets(
src_targets, ref_targets, app, resource_name, src_env, ref_env, saas_file
)
if not src_target:
# this resource was not marked for deployment
log.warn(
"%s -- not marked for deploy in src env, skipping resource!",
_format_app_resource(app, resource_name, saas_file),
)
continue
if resource_name in template_ref_overrides:
# if template ref has explicitly been overridden, use the override
log.debug(
"%s -- overriding template ref",
_format_app_resource(app, resource_name, saas_file),
)
template_ref = template_ref_overrides[resource_name]
elif not ref_target:
# if template ref not overridden, and there's no ref target, we don't know what git ref
# to use for template download
log.warn(
"%s -- no ref target found nor template ref override given, defaulting to 'master'",
_format_app_resource(app, resource_name, saas_file),
)
template_ref = "master"
else:
# otherwise use template ref configured in the "reference deploy target"
template_ref = ref_target["ref"]
raw_template = _download_raw_template(resource, template_ref)
# merge the various layers of parameters to pass into the template
p = copy.deepcopy(json.loads(src_env_data["parameters"]))
p.update(saas_file["parameters"])
p.update(resource["parameters"])
p.update(src_target["parameters"])
# set IMAGE_TAG to be the reference env's IMAGE_TAG
p["IMAGE_TAG"] = ref_target.get("parameters", {}).get("IMAGE_TAG")
if not p.get("IMAGE_TAG"):
if template_ref == "master" or template_ref == "main":
image_tag = "latest"
else:
image_tag = template_ref[:7]
p.update({"IMAGE_TAG": image_tag})
log.debug(
"%s -- no IMAGE_TAG found on ref target, assuming tag '%s' based on template ref",
_format_app_resource(app, resource_name, saas_file),
p["IMAGE_TAG"],
)
# set the env name based on the namespace you intend to deploy to
p["ENV_NAME"] = conf.ENV_NAME_FORMAT.format(namespace=namespace)
processed_template = process_template(raw_template, p)
items.extend(processed_template.get("items", []))
return items
