def _validate_component_dict(all_components, data, name):
# Translate deprecated "APP_NAME/COMPONENT_NAME" path syntax
updated_data = {}
# Max number of items we expect separated by '/' in the CLI option value
max_len = 2
if name == "--set-parameter":
max_len = 3
for path, value in data.items():
split = path.split("/")
if len(split) == max_len:
# first item was an app name
new_path = split[1:]
elif len(split) == max_len - 1:
# first item was a component name
new_path = split[0:]
else:
raise FatalError(f"invalid format for {name}: {path}={value}")
component_name = new_path[0]
# Make sure component name actually exists in app config
if component_name not in all_components:
raise FatalError(
f"component given for {name} not found in app config: {component_name}"
)
key = "/".join(new_path)
updated_data[key] = value
# Update the paths
data.clear()
for key, val in updated_data.items():
data[key] = val
def process_clowd_env(target_ns, quay_user, env_name, template_path):
log.info("processing ClowdEnvironment")
env_template_path = Path(template_path if template_path else conf.DEFAULT_CLOWDENV_TEMPLATE)
if not env_template_path.exists():
raise FatalError("ClowdEnvironment template file does not exist: %s", env_template_path)
with env_template_path.open() as fp:
template_data = yaml.safe_load(fp)
params = dict()
params["ENV_NAME"] = env_name
if quay_user:
quay_user = quay_user.replace("_", "-")
params["PULL_SECRET_NAME"] = f"{quay_user}-pull-secret"
if target_ns:
params["NAMESPACE"] = target_ns
processed_template = process_template(template_data, params=params)
if not processed_template.get("items"):
raise FatalError("Processed ClowdEnvironment template has no items")
return processed_template
def _delete_reservation(name, namespace, requester):
def _err_handler(err):
msg = f"reservation deletion failed: {str(err)}"
_error(msg)
try:
res = get_reservation(name, namespace, requester)
if res:
_warn_before_delete()
res_name = res["metadata"]["name"]
log.info("deleting reservation '%s'", res_name)
oc("delete", "reservation", res_name)
log.info("reservation '%s' deleted", res_name)
else:
raise FatalError("Reservation lookup failed")
except KeyboardInterrupt as err:
log.error("aborted by keyboard interrupt!")
_err_handler(err)
except TimedOutError as err:
log.error("hit timeout error: %s", err)
_err_handler(err)
except FatalError as err:
log.error("hit fatal error: %s", err)
_err_handler(err)
except Exception as err:
log.exception("hit unexpected error!")
_err_handler(err)
def _extend_reservation(name, namespace, requester, duration):
def _err_handler(err):
msg = f"reservation extension failed: {str(err)}"
_error(msg)
try:
res = get_reservation(name, namespace, requester)
if res:
res_config = process_reservation(
res["metadata"]["name"],
res["spec"]["requester"],
duration,
)
log.debug("processed reservation:\n%s", res_config)
apply_config(None, list_resource=res_config)
else:
raise FatalError("Reservation lookup failed")
except KeyboardInterrupt as err:
log.error("aborted by keyboard interrupt!")
_err_handler(err)
except TimedOutError as err:
log.error("hit timeout error: %s", err)
_err_handler(err)
except FatalError as err:
log.error("hit fatal error: %s", err)
_err_handler(err)
except Exception as err:
log.exception("hit unexpected error!")
_err_handler(err)
else:
log.info("reservation '%s' extended by '%s'", res["metadata"]["name"],
duration)
def _get_component_config(self, component_name):
for _, app_cfg in self.apps_config.items():
for component in app_cfg["components"]:
if component["name"] == component_name:
return component
else:
raise FatalError(f"component with name '{component_name}' not found")
def process_iqe_cji(
clowd_app_name,
debug=False,
marker="",
filter="",
env="clowder_smoke",
image_tag="",
cji_name=None,
template_path=None,
requirements="",
requirements_priority="",
test_importance="",
):
log.info("processing IQE ClowdJobInvocation")
template_path = Path(template_path if template_path else conf.DEFAULT_IQE_CJI_TEMPLATE)
if not template_path.exists():
raise FatalError("CJI template file does not exist: %s", template_path)
with template_path.open() as fp:
template_data = yaml.safe_load(fp)
requirements = requirements.split(",") if requirements else []
requirements_priority = requirements_priority.split(",") if requirements_priority else []
test_importance = test_importance.split(",") if test_importance else []
params = dict()
params["DEBUG"] = str(debug).lower()
params["MARKER"] = marker
params["FILTER"] = filter
params["ENV_NAME"] = env
params["IMAGE_TAG"] = image_tag
params["NAME"] = cji_name or f"iqe-{str(uuid.uuid4()).split('-')[0]}"
params["APP_NAME"] = clowd_app_name
params["REQUIREMENTS"] = json.dumps(requirements)
params["REQUIREMENTS_PRIORITY"] = json.dumps(requirements_priority)
params["TEST_IMPORTANCE"] = json.dumps(test_importance)
processed_template = process_template(template_data, params=params)
if not processed_template.get("items"):
raise FatalError("Processed CJI template has no items")
return processed_template
def _fetch_apps_file(config):
rf = RepoFile.from_config(config["appsFile"])
commit, content = rf.fetch()
log.info(
"loading commit '%s' of %s repo %s/%s at path '%s' for apps config",
commit,
rf.host,
rf.org,
rf.repo,
rf.path,
)
fetched_apps = yaml.safe_load(content)
if "apps" not in fetched_apps:
raise FatalError("fetched apps file has no 'apps' key")
app_names = [a["name"] for a in fetched_apps["apps"]]
dupes = get_dupes(app_names)
if dupes:
raise FatalError("duplicate app names found in fetched apps file: {dupes}")
return {a["name"]: a for a in fetched_apps["apps"]}
def import_secrets_from_dir(path):
if not os.path.exists(path):
raise FatalError(f"secrets directory not found: {path}")
if not os.path.isdir(path):
raise FatalError(f"invalid secrets directory: {path}")
files = _get_files_in_dir(path)
secrets = {}
log.info("importing secrets from local path: %s", path)
for secret_file in files:
secrets_in_file = _parse_secret_file(secret_file)
log.info("loaded %d secret(s) from file '%s'", len(secrets_in_file),
secret_file)
for secret_name in secrets_in_file:
if secret_name in secrets:
raise FatalError(
f"secret with name '{secret_name}' defined twice in secrets dir"
)
secrets.update(secrets_in_file)
for secret_name, secret_data in secrets.items():
_import_secret(secret_name, secret_data)
def _create_new_reservation(bot, name, requester, duration, timeout):
def _err_handler(err):
msg = f"reservation failed: {str(err)}"
_error(msg)
try:
res = get_reservation(name)
# Name should be unique on reservation creation.
if res:
raise FatalError(f"Reservation with name {name} already exists")
res_config = process_reservation(name, requester, duration)
log.debug("processed reservation:\n%s", res_config)
if not bot:
if check_for_existing_reservation(
res_config["items"][0]["spec"]["requester"]):
_warn_of_existing(res_config["items"][0]["spec"]["requester"])
try:
res_name = res_config["items"][0]["metadata"]["name"]
except (KeyError, IndexError):
raise Exception(
"error parsing name of Reservation from processed template, "
"check Reservation template")
apply_config(None, list_resource=res_config)
ns_name = wait_on_reservation(res_name, timeout)
except KeyboardInterrupt as err:
log.error("aborted by keyboard interrupt!")
_err_handler(err)
except TimedOutError as err:
log.error("hit timeout error: %s", err)
_err_handler(err)
except FatalError as err:
log.error("hit fatal error: %s", err)
_err_handler(err)
except Exception as err:
log.exception("hit unexpected error!")
_err_handler(err)
else:
log.info(
"namespace '%s' is reserved by '%s' for '%s'",
ns_name,
res_config["items"][0]["spec"]["requester"],
duration,
)
click.echo(ns_name)
def _find_dupe_components(components_for_app):
"""Make sure no component is listed more than once across all apps."""
for app_name, components in components_for_app.items():
components_for_other_apps = copy.copy(components_for_app)
del components_for_other_apps[app_name]
for component in components:
found_in = [app_name]
for other_app_name, other_components in components_for_other_apps.items():
if component in other_components:
found_in.append(other_app_name)
if len(found_in) > 1:
raise FatalError(
f"component '{component}' is not unique, found in apps: {found_in}"
)
def process_reservation(
name,
requester,
duration,
template_path=None,
):
log.info("processing namespace reservation")
template_path = Path(template_path if template_path else conf.DEFAULT_RESERVATION_TEMPLATE)
if not template_path.exists():
raise FatalError("Reservation template file does not exist: %s", template_path)
with template_path.open() as fp:
template_data = yaml.safe_load(fp)
params = dict()
params["NAME"] = name if name else f"bonfire-reservation-{str(uuid.uuid4()).split('-')[0]}"
params["DURATION"] = duration
if requester is None:
try:
requester = whoami()
except Exception:
log.info("whoami returned an error - setting requester to 'bonfire'") # minikube
requester = "bonfire"
params["REQUESTER"] = requester
processed_template = process_template(template_data, params=params)
if not processed_template.get("items"):
raise FatalError("Processed Reservation template has no items")
return processed_template
def _sub_params(self, current_component_name, params):
for path, value in self.param_overrides.items():
split = path.split("/")
if len(split) == 2:
component_name, param_name = split
else:
raise FatalError(f"invalid format for --set-parameter: {path}={value}")
if current_component_name == component_name:
log.info(
"component: '%s' overriding param '%s' to '%s'",
component_name,
param_name,
value,
)
params[param_name] = value
def load_config(config_path=None):
if config_path:
log.debug("user provided explicit config path: %s", config_path)
config_path = Path(config_path)
if not config_path.exists():
raise FatalError(f"provided config file path '{str(config_path)}' does not exist")
else:
# no user-provided path, check default locations
config_path = Path("config.yaml")
if not config_path.exists():
log.debug("./config.yaml not found, using default path: %s", DEFAULT_CONFIG_PATH)
config_path = DEFAULT_CONFIG_PATH
if not config_path.exists():
write_default_config()
log.info("default config not found, creating")
log.info("using local config file: %s", str(config_path.absolute()))
local_config_data = load_file(config_path)
return local_config_data
def _parse_secret_file(path):
"""
Return a dict of all secrets in a file with key: secret name, val: parsed secret json/yaml
The file can contain 1 secret, or a list of secrets
"""
content = load_file(path)
secrets = {}
if content.get("kind").lower() == "list":
items = content.get("items", [])
else:
items = [content]
for item in items:
if item.get("kind").lower() == "secret":
try:
secrets[item["metadata"]["name"]] = item
except KeyError:
raise FatalError(
"Secret at path '{}' has no metadata/name".format(path))
return secrets
def _get_component_items(self, component_name):
component = self._get_component_config(component_name)
try:
rf = RepoFile.from_config(component)
# override template ref if requested
self._sub_ref(component_name, rf)
commit, template_content = rf.fetch()
except Exception as err:
log.error("failed to fetch template file for %s", component_name)
log.debug(traceback.format_exc())
raise FatalError(err)
template = yaml.safe_load(template_content)
params = {
"IMAGE_TAG": commit[:7],
"ENV_NAME": self.clowd_env,
}
params.update(component.get("parameters", {}))
# override any specific parameters on this component if requested
self._sub_params(component_name, params)
new_items = process_template(template, params)["items"]
# override the tags for all occurences of an image if requested
new_items = self._sub_image_tags(new_items)
if (
"all" not in self.no_remove_resources
and ("all" in self.remove_resources or component_name in self.remove_resources)
and component_name not in self.no_remove_resources
):
_remove_resource_config(new_items)
if self.single_replicas:
_set_replicas(new_items)
return new_items
def _parse_apps_in_cfg(config):
app_names = [a["name"] for a in config["apps"]]
dupes = get_dupes(app_names)
if dupes:
raise FatalError("duplicate app names found in config: {dupes}")
return {a["name"]: a for a in config["apps"]}
def _validate_component_list(all_components, data, name):
for component_name in data:
if component_name not in all_components:
raise FatalError(
f"component given for {name} not found in app config: {component_name}"
)
def _validate(self):
"""
Validate app configurations and options passed to the TemplateProcessor
1. Check that each app has required keys
2. Check that each app name is unique
3. Check that each component in an app has required keys
4. Check that each component is a unique name across the whole config
5. Check that CLI params requiring a component use a valid component name
"""
components_for_app = {}
for app_name, app_cfg in self.apps_config.items():
# Check that each app has required keys
required_keys = ["name", "components"]
missing_keys = [k for k in required_keys if k not in app_cfg]
if missing_keys:
raise FatalError(f"app '{app_name}' is missing required keys: {missing_keys}")
# Check that each app name is unique
app_name = app_cfg["name"]
if app_name in components_for_app:
raise FatalError(f"app with name '{app_name}' is not unique")
components_for_app[app_name] = []
for component in app_cfg.get("components", []):
# Check that each component in an app has required keys
required_keys = ["name", "host", "repo", "path"]
missing_keys = [k for k in required_keys if k not in component]
if missing_keys:
raise FatalError(
f"component on app {app_name} is missing required keys: {missing_keys}"
)
comp_name = component["name"]
components_for_app[app_name].append(comp_name)
# Check that each component name is unique across the whole config
self._find_dupe_components(components_for_app)
# Check that CLI params requiring a component use a valid component name
all_components = []
for _, app_components in components_for_app.items():
all_components.extend(app_components)
log.debug("components found: %s", all_components)
self._validate_component_options(
all_components, self.template_ref_overrides, "--set-template-ref"
)
self._validate_component_options(all_components, self.param_overrides, "--set-parameter")
# 'all' is a valid component keyword for these options below
all_components.append("all")
self._validate_component_options(
all_components, self.remove_resources, "--remove-resources"
)
self._validate_component_options(
all_components, self.no_remove_resources, "--no-remove-resources"
)
self._validate_component_options(all_components, self.component_filter, "--component")
def _get_app_config(self, app_name):
if app_name not in self.apps_config:
raise FatalError(f"app {app_name} not found in apps config")
return self.apps_config[app_name]