def __init__(self, vars=None, secret_keys=None):
super(VarsInventory, self).__init__()
if vars is None:
vars = []
if not isinstance(vars, Sequence):
vars = [vars]
self._init_vars_list = vars
if secret_keys is None:
secret_keys = []
self._secrets = secret_keys
self._vars = {}
for v in self._init_vars_list:
dict_merge(self._vars, v, copy_dct=False)
def add_result(self, result_id, result):
try:
registered = self._registers.get(result_id, None)
if registered is None:
raise FrklException(
msg=
"Result for id '{}' not registered, this is most likely a bug."
.format(result_id))
value_template = registered.get("value", None)
if value_template is not None:
new_value = replace_strings_in_obj(
value_template,
replacement_dict={"__result__": result},
jinja_env=DEFAULT_RUN_CONFIG_JINJA_ENV,
)
else:
new_value = result
target = registered.get("target", None)
if target is None:
if not isinstance(result, Mapping):
raise FrklException(
msg="Can't merge result id '{}'".format(result_id),
reason=
"Value for result-id '{}' not a mapping, and no 'target' provided.",
solution=
"Either provide a 'target' value, or use the 'value' template to convert the result.",
)
dict_merge(self._result, new_value, copy_dct=False)
else:
temp = {}
dpath.new(temp, target, new_value, separator=".")
dict_merge(self._result, temp, copy_dct=False)
except (Exception) as e:
log.error("Could not register result '{}': {}".format(
result_id, e))
def set_run_defaults(inventory=None, run_config=None, run_vars=None):
if inventory is None:
inventory = VarsInventory()
if isinstance(inventory, Mapping):
inventory = VarsInventory(vars=inventory)
if run_config is None:
run_config = FrecklesRunConfig()
if isinstance(run_config, string_types):
run_config = FrecklesRunConfig(target_string=run_config)
if isinstance(run_config, FrecklesRunConfig):
run_config = run_config.config
default_run_config = {"host": "localhost"}
run_config = dict_merge(default_run_config, run_config, copy_dct=False)
if run_vars is None:
run_vars = {}
return inventory, run_config, run_vars
def _init_config(self):
if self._target_string is not None:
self._target_dict_base = parse_target_string(self._target_string)
else:
self._target_dict_base = {}
self._config = dict_merge(self._target_dict_base,
self._target_dict,
copy_dct=False)
if "host" not in self._config.keys():
self._config["host"] = "localhost"
self._protocol = self._config.get("protocol", None)
self._user = self._config.get("user", None)
self._port = self._config.get("port", None)
self._host = self._config.get("host", None)
self._connection_type = self._config.get("connection_type", None)
self._ssh_key = self._config.get("ssh_key", None)
self._become_pass = self._config.get("become_pass", None)
self._login_pass = self._config.get("login_pass", None)
if self._connection_type == "lxd":
if self._user is None:
self._user = "******"
if self._user != "root":
if not self._warning_showed:
log.warning(
"lxd connection type does not support different user than 'root', ignoring specified user: {}"
.format(self._user))
self._warning_showed = True
self._user = "******"
self._invalidated = False
def run_frecklecutable(
self,
inventory=None,
run_config=None,
run_vars=None,
parent_task=None,
result_callback=None,
elevated=None,
env_dir=None,
password_callback=None,
extra_callbacks=None,
):
if password_callback is None:
password_callback = ask_password
inventory, run_config, run_vars = set_run_defaults(
inventory=inventory, run_config=run_config, run_vars=run_vars)
run_inventory, secret_args = self.create_run_inventory(
inventory=inventory, run_config=run_config, run_vars=run_vars)
if parent_task is None:
i_am_root = True
result_callback = FrecklesResultCallback()
else:
i_am_root = False
if result_callback is None:
raise Exception("No result callback. This is a bug")
# paused = False
# if parent_task is not None and (
# secret_args
# or run_config.get("become_pass", None) == "::ask::"
# or run_config.get("login_pass", None) == "::ask::"
# ):
# paused = True
asked = False
run_secrets = {}
# if parent_task is not None:
# parent_task.pause()
run_secrets["become_pass"] = run_config.pop("become_pass", None)
if (not run_config.get("no_run", False)
and run_secrets["become_pass"] == "::ask::"):
msg = ""
if run_config.get("user", None):
msg = "{}@".format(run_config["user"])
msg = msg + run_config.get("host", "localhost")
prompt = "SUDO PASS (for '{}')".format(msg)
run_secrets["become_pass"] = password_callback(prompt)
asked = True
run_secrets["login_pass"] = run_config.pop("login_pass", None)
if (not run_config.get("no_run", False)
and run_secrets["login_pass"] == "::ask::"):
msg = ""
if run_config.get("user", None):
msg = "{}@".format(run_config["user"])
msg = msg + run_config.get("host", "localhost")
prompt = "LOGIN/SSH PASS (for '{}')".format(msg)
run_secrets["login_pass"] = password_callback(prompt)
asked = True
# if paused:
# parent_task.resume()
if asked:
click.echo()
frecklet_name = self.frecklet.id
log.debug("Running frecklecutable: {}".format(frecklet_name))
tasks = self.process_tasks(inventory=run_inventory)
current_tasklist = []
idempotent_cache = []
current_adapter = None
# all_resources = {}
tasks_elevated = False
task_lists = []
for task in tasks:
elv = (task[FRECKLET_KEY_NAME].get(
FRECKLES_PROPERTIES_METADATA_KEY,
{}).get(FRECKLES_PROPERTIES_ELEVATED_METADATA_KEY, False))
# just converting from string to boolean
if isinstance(elv, string_types):
if elv.lower() in ["true", "1", "yes"]:
elv = True
task[FRECKLET_KEY_NAME][FRECKLES_PROPERTIES_METADATA_KEY][
FRECKLES_PROPERTIES_ELEVATED_METADATA_KEY] = True
if elv:
tasks_elevated = True
tt = task[FRECKLET_KEY_NAME]["type"]
adapter_name = self.context._adapter_tasktype_map.get(tt, None)
if adapter_name is None:
raise Exception(
"No adapter registered for task type: {}".format(tt))
if len(adapter_name) > 1:
raise Exception(
"Multiple adapters registered for task type '{}', that is not supported yet."
.format(tt))
adapter_name = adapter_name[0]
if current_adapter is None:
current_adapter = adapter_name
if current_adapter != adapter_name:
if elevated is not None:
tasks_elevated = elevated
new_tasklist = {
"tasklist": current_tasklist,
"adapter": current_adapter,
"elevated": tasks_elevated,
}
if tasks_elevated:
self.check_become_pass(
run_config,
run_secrets,
parent_task,
password_callback=password_callback,
)
task_lists.append(new_tasklist)
current_adapter = adapter_name
idempotent_cache = []
current_tasklist = []
tasks_elevated = False
if is_duplicate_task(task, idempotent_cache):
log.debug("Idempotent, duplicate task, ignoring: {}".format(
task[FRECKLET_KEY_NAME]["name"]))
continue
current_tasklist.append(task)
if elevated is not None:
tasks_elevated = elevated
new_tasklist = {
"tasklist": current_tasklist,
"adapter": current_adapter,
"elevated": tasks_elevated,
}
if tasks_elevated:
self.check_become_pass(
run_config,
run_secrets,
parent_task,
password_callback=password_callback,
)
task_lists.append(new_tasklist)
current_run_result = None
root_task = None
run_env_properties = None
try:
for run_nr, tl_details in enumerate(task_lists):
current_adapter = tl_details["adapter"]
current_tasklist = tl_details["tasklist"]
run_elevated = tl_details["elevated"]
if not current_tasklist:
continue
# augmenting result properties
for t in current_tasklist:
if "register" in t.get(FRECKLET_KEY_NAME, {}).keys():
result_callback.register_task(t[FRECKLET_KEY_NAME])
adapter = self.context._adapters[current_adapter]
run_env_properties = self.context.create_run_environment(
adapter, env_dir=env_dir)
run_env_properties["frecklet_name"] = self.frecklet.id
run_env_properties["run_metadata"] = run_config.get(
"metadata", {})
# preparing execution environment...
self._context._run_info.get("prepared_execution_environments",
{}).get(current_adapter, None)
if extra_callbacks:
if not isinstance(extra_callbacks, Sequence):
extra_callbacks = [extra_callbacks]
cbs = extra_callbacks + self._callbacks
else:
cbs = self._callbacks
if self.context.config_value("write_run_log"):
c_config = {
"path":
os.path.join(run_env_properties["env_dir"],
"run_log.json")
}
log_file_callback = load_callback("logfile",
callback_config=c_config)
cbs = cbs + [log_file_callback]
if parent_task is None:
root_task = Tasks(
"env_prepare_adapter_{}".format(adapter_name),
msg="starting run",
category="run",
callbacks=cbs,
is_utility_task=False,
)
parent_task = root_task.start()
prepare_root_task = parent_task.add_subtask(
task_name="env_prepare_adapter_{}".format(adapter_name),
msg="preparing adapter: {}".format(adapter_name),
)
try:
adapter.prepare_execution_requirements(
run_config=run_config,
task_list=current_tasklist,
parent_task=prepare_root_task,
)
prepare_root_task.finish(success=True)
except (Exception) as e:
prepare_root_task.finish(success=False, error_msg=str(e))
raise e
host = run_config["host"]
if adapter_name == "freckles":
msg = "running frecklecutable: {}".format(frecklet_name)
else:
msg = "running frecklet: {} (on: {})".format(
frecklet_name, host)
root_run_task = parent_task.add_subtask(
task_name=frecklet_name, msg=msg)
run_config["elevated"] = run_elevated
run_vars = dict_merge(result_callback.result,
run_vars,
copy_dct=True)
if not i_am_root:
r_tks = get_template_keys(
run_config, jinja_env=DEFAULT_RUN_CONFIG_JINJA_ENV)
if r_tks:
for k in r_tks:
if k not in result_callback.result.keys():
raise Exception(
"Could not find result key for subsequent run: {}"
.format(k))
run_config = replace_strings_in_obj(
run_config,
replacement_dict=result_callback.result,
jinja_env=DEFAULT_RUN_CONFIG_JINJA_ENV,
)
run_properties = None
try:
write_runs_log(
properties=run_env_properties,
adapter_name=adapter_name,
state="started",
)
# from random import randint
# from time import sleep
# sec = randint(1, 10)
# print("SLEEPING: {}".format(sec))
# sleep(sec)
run_properties = adapter._run(
tasklist=current_tasklist,
run_vars=run_vars,
run_config=run_config,
run_secrets=run_secrets,
run_env=run_env_properties,
result_callback=result_callback,
parent_task=root_run_task,
)
if not root_run_task.finished:
root_run_task.finish()
run_result = FrecklesRunRecord(
run_id=run_nr,
adapter_name=adapter_name,
task_list=current_tasklist,
run_vars=run_vars,
run_config=run_config,
run_env=run_env_properties,
run_properties=run_properties,
result=copy.deepcopy(result_callback.result),
success=root_run_task.success,
root_task=root_run_task,
parent_result=current_run_result,
)
current_run_result = run_result
write_runs_log(
properties=run_env_properties,
adapter_name=adapter_name,
state="success",
)
if not root_run_task.success:
break
except (Exception) as e:
# import traceback
# traceback.print_exc()
write_runs_log(
properties=run_env_properties,
adapter_name=adapter_name,
state="failed",
)
if isinstance(e, FrklException):
msg = e.message
else:
msg = str(e)
if not root_run_task.finished:
root_run_task.finish(success=False, error_msg=msg)
# click.echo("frecklecutable run failed: {}".format(e))
log.debug(e, exc_info=1)
run_result = FrecklesRunRecord(
run_id=run_nr,
adapter_name=adapter_name,
task_list=current_tasklist,
run_vars=run_vars,
run_config=run_config,
run_env=run_env_properties,
run_properties=run_properties,
result=copy.deepcopy(result_callback.result),
success=root_run_task.success,
root_task=root_run_task,
parent_result=current_run_result,
exception=e,
)
current_run_result = run_result
break
except (Exception) as e:
log.error(e)
finally:
if root_task is None:
return current_run_result
if i_am_root:
root_task.finish()
keep_run_folder = self.context.config_value(
"keep_run_folder", "context")
if i_am_root and not keep_run_folder:
env_dir = run_env_properties["env_dir"]
env_dir_link = run_env_properties.get("env_dir_link", None)
if env_dir_link and os.path.realpath(env_dir_link) == env_dir:
try:
log.debug("removing env dir symlink: {}".format(
env_dir_link))
os.unlink(env_dir_link)
except (Exception):
log.debug("Could not remove symlink.")
try:
log.debug("removing env dir: {}".format(env_dir))
shutil.rmtree(env_dir)
except (Exception) as e:
log.warning(
"Could not remove environment folder '{}': {}".format(
env_dir, e))
if current_run_result:
result_dict = current_run_result.result
success = current_run_result.success
else:
success = False
if success:
if not result_dict:
result_dict = {}
else:
result_dict = special_dict_to_dict(result_dict)
for cb in self.context.result_callback:
if cb[0] == "pretty":
click.echo()
if result_dict:
result_yaml = readable(result_dict,
out="yaml",
indent=4)
msg = (Style.BRIGHT + "Result:" + Style.RESET_ALL +
"\n\n" + Style.DIM + result_yaml +
Style.RESET_ALL)
output_to_terminal(msg)
click.echo()
elif cb[0] == "json":
result_json = json.dumps({
"output_type": "freckles_run_result",
"value": result_dict
})
output_to_terminal(result_json, nl=True)
elif str(cb[0].lower()) in ["false", "no", "silent"]:
continue
else:
log.warning("Result callback '{}' not supported.".format(
cb[0]))
return current_run_result
def describe_frecklet(context, frecklet, vars, auto_vars=False):
fx = frecklet.create_frecklecutable(context=context)
var_all = {}
if isinstance(vars, Mapping):
vars = [vars]
for v in vars:
dict_merge(var_all, v, copy_dct=False)
auto_vars_dict = None
if auto_vars:
params = fx.frecklet.vars
auto_vars_dict = create_auto_vars(params,
existing_vars=var_all,
frecklet=fx.frecklet)
var_all = dict_merge(auto_vars_dict, var_all, copy_dct=True)
inv = VarsInventory(var_all)
run_inventory, secret_vars = fx.create_run_inventory(inventory=inv)
tasks = fx.process_tasks(inventory=run_inventory)
result = []
for task in tasks:
# output(task, output_type="yaml")
f_type = task[FRECKLET_KEY_NAME]["type"]
name = task[FRECKLET_KEY_NAME]["name"]
f_doc = Doc(
task[FRECKLET_KEY_NAME].get(FRECKLES_DESC_METADATA_KEY, {}),
short_help_key=FRECKLES_DESC_SHORT_METADATA_KEY,
help_key=FRECKLES_DESC_LONG_METADATA_KEY,
further_reading_key=FRECKLES_DESC_REFERENCES_METADATA_KEY,
)
title = f_doc.get_short_help(default=None,
use_help=False,
list_item_format=True)
if title:
if title.startswith("["):
title = title[1:]
if title.endswith("]"):
title = title[0:-1]
else:
title = None
if f_type == "frecket":
alt_title = context.get_frecklet(name).doc.get_short_help(
list_item_format=True)
alt_desc = context.get_frecklet(name).doc.get_short_help(
list_item_format=False)
if alt_desc:
alt_desc = alt_desc.strip()
else:
alt_title = "execute {}: {}".format(f_type, name)
alt_desc = "Run {} '{}'".format(f_type, name)
desc = f_doc.get_help(default=None, use_short_help=False)
if desc:
desc = desc.strip()
if not task.get(VARS_KEY):
alt_desc = alt_desc + " (no variables used)."
else:
vars_string = readable(task[VARS_KEY], out="yaml")
alt_desc = alt_desc + " using variables:\n\n" + reindent(
vars_string, 2)
alt_desc = alt_desc.strip()
if task.get(TASK_KEY_NAME, None):
task_string = readable(task[TASK_KEY_NAME], out="yaml")
alt_desc = (alt_desc + "\n\nand task metadata:\n\n" +
reindent(task_string, 2))
alt_desc = alt_desc.strip()
task_md = {}
task_md["title"] = title
task_md["alt_title"] = alt_title
task_md["desc"] = desc
task_md["alt_desc"] = alt_desc
task_md["references"] = f_doc.get_further_reading()
result.append(task_md)
return result, auto_vars_dict
def create_parameter(self, var_name, var):
if not var.cli.get("enabled", True):
return None
option_properties = dict_merge(
CliArgumentsAttribute.DEFAULT_CLI_SCHEMA, var.cli, copy_dct=True)
param_type = option_properties.pop("param_type")
if not var.required:
option_properties["required"] = False
else:
if var.schema.get("excludes", []):
option_properties["required"] = False
else:
option_properties["required"] = True
auto_param_decls = False
if "param_decls" not in option_properties.keys():
auto_param_decls = True
if param_type == "option":
decls = []
for a in var.aliases:
if len(a) == 1:
decls.append("-{}".format(a))
else:
a = a.replace("_", "-")
decls.append("--{}".format(a))
option_properties["param_decls"] = decls
else:
option_properties["param_decls"] = [var.key]
if "metavar" not in option_properties.keys():
option_properties["metavar"] = var.key.upper()
# setting type
cerberus_type = var.type
if not isinstance(cerberus_type, string_types) and isinstance(
cerberus_type, Sequence):
replacement = None
cerberus_type = "multi"
else:
replacement = FRECKLES_CLICK_CEREBUS_ARG_MAP.get(
cerberus_type, None)
if replacement is not None:
if replacement == bool:
option_properties["type"] = None
option_properties["default"] = None
if "is_flag" not in option_properties.keys():
option_properties["is_flag"] = True
# we don't add the type here, otherwise click fails for whatever reason
if "param_decls" not in option_properties.keys(
) or auto_param_decls:
temp = var.key.replace("_", "-")
option_properties["param_decls"] = [
"--{}/--no-{}".format(temp, temp)
]
else:
option_properties["type"] = replacement
elif cerberus_type == "list":
arg_schema = var.schema.get("schema", {})
schema_type = arg_schema.get("type", "string")
replacement = FRECKLES_CLICK_CEREBUS_ARG_MAP.get(
schema_type, click.STRING)
option_properties["type"] = replacement
if param_type == "option":
option_properties["multiple"] = True
else:
if ("nargs" not in option_properties.keys()
and "default" not in option_properties.keys()):
option_properties["nargs"] = -1
elif cerberus_type != "multi":
raise Exception(
"Type '{}' not implemented yet.".format(cerberus_type))
# if var.secret:
# if "default" not in option_properties.keys():
# if option_properties["required"]:
# option_properties["default"] = "::ask::"
# option_properties["show_default"] = True
if option_properties.get("default", None) is not None:
default_val = option_properties["default"]
if is_var_adapter(default_val):
option_properties["type"] = str
if var.secret:
option_properties["show_default"] = False
if param_type == "option":
option_properties["help"] = var.doc.get_short_help()
p = click.Option(**option_properties)
else:
option_properties.pop("show_default", None)
if ("nargs" in option_properties.keys()
and "default" in option_properties.keys()):
log.warning(
"Removing 'nargs' property from argument '{}' ('nargs' & 'default' are not allowed together)"
.format(var_name))
option_properties.pop("nargs")
p = click.Argument(**option_properties)
return p
def resolve_vars(self, current_args, rest_path, last_node, tree, ting,
is_root_level):
current_node_id = next(rest_path)
current_node = tree.get_node(current_node_id)
if current_node_id == 0:
vars = {}
for key in current_args.keys():
vars[key] = "{{{{:: {} ::}}}}".format(key)
else:
vars = current_node.data["task"]["vars"]
if current_node_id == 0:
available_args = current_node.data.args
else:
available_args = current_node.data["task"]["args"]
args = {}
for key, arg in current_args.items():
# if key in ting.const.keys():
# parent_var = ting.const[key]
# print("---")
# print(parent_var)
if key in vars.keys():
parent_var = vars[key]
elif key not in vars.keys():
inherit = ting.meta.get("vars", {}).get("inherit", False)
if inherit:
vars[key] = "{{{{:: {} ::}}}}".format(key)
parent_var = vars[key]
else:
if arg.required and arg.default:
# relevant frecklet root name: tree.get_node(0).tag,
raise FreckletBuildException(
frecklet=ting,
msg=
"Invalid frecklet-task '{}' in '{}': does not provide required/non-defaulted var '{}' for child frecklet '{}'."
.format(
current_node.tag,
tree.get_node(0).tag,
key,
last_node.tag,
),
solution="Check format of frecklet '{}'.".format(
current_node.tag),
references={
"frecklet documentation":
"https://freckles.io/doc/frecklets/anatomy"
},
)
else:
continue
new_args = self.create_new_args_map(parent_var=parent_var,
key=key,
available_args=available_args,
arg=arg)
for k, v in new_args.items():
# if k in args.keys():
# log.warning("Duplicate arg key: {}".format(k))
args[k] = v
if current_node_id != 0:
r = self.resolve_vars(
current_args=args,
rest_path=rest_path,
last_node=current_node,
tree=tree,
ting=ting,
is_root_level=False,
)
return r
else:
# new_args = {}
# replaced = []
# for key, arg in args.items():
# if key in ting.const.keys():
# replaced.append(key)
# parent_var = ting.const[key]
# temp_args = self.create_new_args_map(parent_var=parent_var, key=key, available_args=available_args, arg=args[key])
# for k, v in temp_args.items():
# new_args[k] = v
# else:
# new_args[key] = arg
#
# args = new_args
root_task = last_node.data["task"]
try:
tks = get_template_keys(root_task,
jinja_env=DEFAULT_FRECKLES_JINJA_ENV)
except (TemplateSyntaxError) as e:
raise FreckletException(ting, e, ting.id)
# for r in replaced:
# tks.remove(r)
root_tks = {}
for tk in tks:
if tk in args.keys():
continue
# add a template key that is under either 'frecklet' or 'task' of the root task of a frecklet
arg = available_args.get(tk, None)
auto_arg = False
if arg is None:
if self.default_argument_description is None:
f_name = current_node.data.id
raise Exception(
"No argument description for argument '{}' in frecklet '{}'"
.format(tk, f_name))
else:
arg = copy.copy(self.default_argument_description)
auto_arg = True
arg = Arg(
tk,
arg,
default_schema=FRECKLES_DEFAULT_ARG_SCHEMA,
is_auto_arg=auto_arg,
)
root_tks[tk] = arg
dict_merge(args, root_tks, copy_dct=False)
return args