def create_dotfiles_packages(self, profile_vars):
result = []
for folder, subfolder_list in profile_vars.items():
for subfolder_metadata in subfolder_list:
md = {}
md["stow_source"] = subfolder_metadata[
'freckles_app_dotfile_folder_path']
md["stow_folder_name"] = subfolder_metadata[
'freckles_app_dotfile_folder_name']
md["name"] = subfolder_metadata[
'freckles_app_dotfile_folder_name']
md["stow_folder_parent"] = subfolder_metadata[
'freckles_app_dotfile_parent_path']
parent_details = subfolder_metadata.get(
'freckles_app_dotfile_parent_details', {})
extra_vars = copy.deepcopy(
parent_details.get("extra_vars", {}).get(md["name"], {}))
package_md = extra_vars.pop("package", None)
overlay = frkl.dict_merge(md, extra_vars)
if package_md:
frkl.dict_merge(overlay, package_md, copy_dct=False)
result.append({"vars": overlay})
return result
def get_vars_from_cli_input(input_args, key_map, task_vars, default_vars, args_that_are_vars, value_vars):
# exchange arg_name with var name
new_args = {}
for key, value in key_map.items():
temp = input_args.pop(key.replace('-', '_'))
if key not in args_that_are_vars:
if isinstance(temp, tuple):
temp = list(temp)
new_args[value] = temp
else:
task_vars[value] = temp
# replace all matching strings in value_vars
for i, var_name in enumerate(value_vars):
if var_name == key:
value_vars[i] = value
# now overimpose the new_args over template_vars
new_args = dict_merge(default_vars, new_args)
final_vars = {}
sub_dict = copy.deepcopy(new_args)
# inject subdict (args and envs) in vars
for key, template in task_vars.items():
if isinstance(template, string_types):
template_var_string = replace_string(template, sub_dict)
if template_var_string.startswith('{') and not \
template_var_string.startswith('{{') and not \
template_var_string.startswith('{%'):
# if template_var_string is json, load value
# (but do not handle {{ ansible-side substitution)
try:
template_var_new = yaml.safe_load(template_var_string)
final_vars[key] = template_var_new
except (Exception) as e:
raise Exception("Could not convert template '{}': {}".format(template_var_string, e.message))
else:
# else push value
final_vars[key] = template_var_string
else:
final_vars[key] = template
new_vars = {}
for key in value_vars:
values = final_vars.pop(key, {})
if values and not isinstance(values, dict):
raise Exception("value for '{}' not a dict: {}".format(key, values))
if values:
dict_merge(new_vars, values, copy_dct=False)
dict_merge(new_vars, final_vars, copy_dct=False)
return new_args, new_vars
def flatten_profile_vars(self, profile_folder_list):
if not profile_folder_list:
return {}
result = {}
for f in profile_folder_list:
frkl.dict_merge(result, f["vars"], copy_dct=False)
return result
def process_folder_vars(self,
folder_vars,
default_vars,
overlay_vars,
base_vars={}):
final_vars = frkl.dict_merge(default_vars, folder_vars, copy_dct=True)
if base_vars:
frkl.dict_merge(final_vars, base_vars, copy_dct=False)
frkl.dict_merge(final_vars, overlay_vars, copy_dct=False)
return final_vars
def process_lines_old(self, content, current_vars):
log.debug("Processing content: {}".format(content))
# now, I know this isn't really the most
# optimal way of doing this,
# but I don't really care that much about execution speed yet,
# plus I really want to be able to use variables used in previous
# lines of the content
last_whitespaces = 0
current_lines = ""
temp_vars = copy.deepcopy(current_vars)
for line in content:
if line.strip().startswith("#"):
continue
whitespaces = len(line) - len(line.lstrip(' '))
current_lines = "{}{}\n".format(current_lines, line)
if whitespaces <= last_whitespaces:
temp = replace_string(current_lines, temp_vars,
**self.delimiter_profile)
if not temp.strip():
continue
temp_dict = ordered_load(temp)
if temp_dict:
temp_vars = frkl.dict_merge(temp_vars,
temp_dict,
copy_dct=False)
last_whitespaces = whitespaces
if current_lines:
temp = replace_string(
current_lines,
temp_vars,
additional_jinja_extensions=freckles_jinja_extensions,
**self.delimiter_profile)
temp_dict = ordered_load(temp)
temp_vars = frkl.dict_merge(temp_vars, temp_dict, copy_dct=False)
frkl.dict_merge(current_vars, temp_vars, copy_dct=False)
log.debug("Vars after processing:\n{}".format(
readable_json(current_vars, indent=2)))
return current_vars
def create_folder_packages(self, package_names, extra_vars):
result = []
for name in package_names:
details = {"name": name}
if name in extra_vars.keys():
no_install = extra_vars.get(name, {}).get("no_install", False)
if no_install:
continue
package_md = extra_vars[name].get("package", None)
if package_md:
frkl.dict_merge(details, package_md, copy_dct=False)
result.append({"vars": details})
return result
def process_dictlet(self, metadata, dictlet_details):
freckles_meta = metadata.get(FX_FRECKLES_META_KEY_NAME, {})
defaults = metadata.get(FX_DEFAULTS_KEY_NAME, {})
doc = metadata.get(FX_DOC_KEY_NAME, {})
c_vars_dictlet = metadata.get(FX_ARGS_KEY_NAME, {})
c_vars_command = self.command.get_additional_args()
# adapter args take precedence
if c_vars_command:
c_vars = frkl.dict_merge(c_vars_dictlet,
c_vars_command,
copy_dct=True)
else:
c_vars = c_vars_dictlet
if not isinstance(c_vars, dict):
c_vars = convert_args_to_dict(c_vars)
params = parse_args_dict(c_vars)
@click.command(cls=FrecklesCliFormatter, name=self.name)
@click.pass_context
def command(ctx, *args, **kwargs):
context_repos = freckles_meta.get("context_repos", [])
if context_repos:
output = ctx.parent.params.get("output", "default")
download_repos(context_repos, self.command.get_config(),
output)
user_input = clean_user_input(kwargs, c_vars)
result = self.command.freckles_process(
self.name,
defaults,
self.extra_vars,
user_input,
metadata,
dictlet_details,
config=self.command.get_config(),
parent_params=self.parent_params,
command_var_spec=c_vars)
return result
command.params = params
help_string = metadata.get(FX_DOC_KEY_NAME, {}).get("help", None)
if help_string:
command.help = help_string
help_details = generate_details(metadata, dictlet_details)
command.freckles_cli_details = help_details
if "short_help" in doc.keys():
command.short_help = doc.get("short_help")
if "epilog" in doc.keys():
command.epilog = doc.get("epilog", None)
return command
def replace_string(template_string, replacement_dict):
use_environment_vars = True
if use_environment_vars:
sub_dict = copy.deepcopy({"LOCAL_ENV": os.environ})
dict_merge(sub_dict, replacement_dict, copy_dct=False)
else:
sub_dict = replacement_dict
trim_blocks = True
block_start_string = '{%::'
block_end_string = '::%}'
variable_start_string = '{{::'
variable_end_string = '::}}'
result = Environment(extensions=[freckles_jinja_utils, ansible_extensions.utils], trim_blocks=trim_blocks, block_start_string=block_start_string, block_end_string=block_end_string, variable_start_string=variable_start_string, variable_end_string=variable_end_string).from_string(template_string).render(sub_dict)
return result
def get_all_dictlets(self):
"""Find all freckelize adapters."""
log.debug("Retrieving all dictlets")
if self.adapter_cache is None:
self.adapter_cache = {}
dictlet_names = OrderedDict()
all_adapters = OrderedDict()
for path in self.paths:
if path not in self.path_cache.keys():
adapters = find_freckelize_adapters(path)
self.path_cache[path] = adapters
frkl.dict_merge(all_adapters, adapters, copy_dct=False)
frkl.dict_merge(self.adapter_cache, adapters, copy_dct=False)
return all_adapters
def get_all_dictlets(self):
"""Find all frecklecutables."""
if self.frecklecutable_cache is None:
self.frecklecutable_cache = {}
all_frecklecutables = OrderedDict()
for path in self.paths:
if path not in self.path_cache.keys():
commands = OrderedDict()
dirs = find_frecklecutable_dirs(path)
for f_dir in dirs:
fx = find_frecklecutables_in_folder(f_dir)
frkl.dict_merge(commands, fx, copy_dct=False)
self.path_cache[path] = commands
frkl.dict_merge(self.frecklecutable_cache,
commands,
copy_dct=False)
frkl.dict_merge(all_frecklecutables,
self.path_cache[path],
copy_dct=False)
return all_frecklecutables
def convert(self, value, param, ctx):
chain = [
UrlAbbrevProcessor(), EnsureUrlProcessor(), EnsurePythonObjectProcessor(), LoadMoreConfigsProcessor()]
try:
if not isinstance(value, (list, tuple)):
value = [value]
frkl_obj = Frkl(value, chain)
result = frkl_obj.process()
if isinstance(result[0], (list, tuple)):
result_dict = {}
for item in result[0]:
dict_merge(result_dict, item, copy_dct=False)
return result_dict
else:
return result[0]
except (Exception) as e:
self.fail("Can't read vars '{}': {}".format(value, str(e)))
def assemble_freckle_run(*args, **kwargs):
try:
result = []
no_run = kwargs.get("no_run")
hosts = list(kwargs["host"])
if not hosts:
hosts = ["localhost"]
default_target = kwargs.get("target_folder", None)
if not default_target:
default_target = DEFAULT_FRECKLE_TARGET_MARKER
default_freckle_urls = list(kwargs.get("freckle", []))
default_output_format = kwargs.get("output", "default")
default_include = list(kwargs.get("include", []))
default_exclude = list(kwargs.get("exclude", []))
default_ask_become_pass = kwargs.get("ask_become_pass", True)
default_extra_vars_raw = list(kwargs.get("extra_vars", []))
default_non_recursive = kwargs.get("non_recursive", False)
if not default_non_recursive:
default_non_recursive = False
default_extra_vars = {}
for ev in default_extra_vars_raw:
dict_merge(default_extra_vars, ev, copy_dct=False)
extra_profile_vars = {}
repos = collections.OrderedDict()
profiles = []
metadata = {}
if not args[0]:
# auto-detect
metadata["__auto__"] = {}
fr = {
"target_folder": default_target,
"includes": default_include,
"excludes": default_exclude,
"password": default_ask_become_pass,
"non_recursive": default_non_recursive
}
for f in default_freckle_urls:
repos[f] = copy.deepcopy(fr)
repos[f]["profiles"] = ["__auto__"]
extra_profile_vars = default_extra_vars
else:
for p in args[0]:
pn = p["name"]
metadata[pn] = p["metadata"]
if pn in profiles:
raise Exception(
"Profile '{}' specified twice. I don't think that makes sense. Exiting..."
.format(pn))
else:
profiles.append(pn)
pvars = p["vars"]
freckles = list(pvars.pop("freckle", []))
include = set(pvars.pop("include", []))
exclude = set(pvars.pop("exclude", []))
target = pvars.pop("target-folder", None)
ask_become_pass = pvars.pop("ask_become_pass", "auto")
non_recursive = pvars.pop("non_recursive", False)
if non_recursive == None:
non_recursive = default_non_recursive
if ask_become_pass == "auto" and default_ask_become_pass != "auto":
ask_become_pass = default_ask_become_pass
extra_profile_vars[pn] = copy.deepcopy(
default_extra_vars.get(pn, {}))
if pvars:
for pk, pv in pvars.items():
if pv != None:
extra_profile_vars[pn][pk] = pv
all_freckles_for_this_profile = list(
set(default_freckle_urls + freckles))
for freckle_url in all_freckles_for_this_profile:
if target:
t = target
else:
t = default_target
for i in default_include:
if i not in include:
include.add(i)
for e in default_exclude:
if e not in exclude:
exclude.add(e)
fr = {
"target_folder": t,
"includes": list(include),
"excludes": list(exclude),
"password": ask_become_pass,
"non_recursive": non_recursive
}
repos.setdefault(freckle_url, fr)
repos[freckle_url].setdefault("profiles", []).append(pn)
if (repos):
click.echo("\n# starting ansible run(s)...")
temp = execute_freckle_run(repos,
profiles,
metadata,
extra_profile_vars=extra_profile_vars,
no_run=no_run,
output_format=default_output_format,
ask_become_pass=default_ask_become_pass,
hosts_list=hosts)
result.append(temp)
click.echo("")
return result
except (Exception) as e:
message = e.message
if not message:
if not e.reason:
message = "n/a"
else:
message = "Reason: {}".format(e.reason)
click.echo(
"\nError assembling configuration.\nMessage: {}\nExiting...".
format(message))
sys.exit(1)
def assemble_freckelize_run(*args, **kwargs):
no_run = kwargs.get("no_run")
hosts = list(kwargs["host"])
if not hosts:
hosts = ["localhost"]
default_target = kwargs.get("target_folder", None)
default_target_name = kwargs.get("target_name", None)
default_freckle_urls = list(kwargs.get("freckle", []))
default_output_format = kwargs.get("output", "default")
default_include = list(kwargs.get("include", []))
default_exclude = list(kwargs.get("exclude", []))
default_password = kwargs.get("password", None)
default_non_recursive = kwargs.get("non_recursive", None)
default_extra_vars_list = list(kwargs.get("vars", []))
default_extra_vars = OrderedDict()
for ev in default_extra_vars_list:
frkl.dict_merge(default_extra_vars, ev, copy_dct=False)
parent_command_vars = {}
if default_target:
parent_command_vars["target_folder"] = default_target
if default_target_name:
parent_command_vars["target_folder_name"] = default_target_name
if default_include:
parent_command_vars["includes"] = default_include
if default_exclude:
parent_command_vars["includes"] = default_include
if default_non_recursive is not None:
parent_command_vars["non_recursive"] = default_non_recursive
freckle_details = []
if not args[0]:
# fill missing keys with default values
if "target_folder" not in parent_command_vars.keys():
parent_command_vars["target_folder"] = DEFAULT_FRECKLE_TARGET_MARKER
if "target_folder_name" not in parent_command_vars.keys():
parent_command_vars["target_folder_name"] = None
if "include" not in parent_command_vars.keys():
parent_command_vars["include"] = []
if "exclude" not in parent_command_vars.keys():
parent_command_vars["exclude"] = []
if "non_recursive" not in parent_command_vars.keys():
parent_command_vars["non_recursive"] = False
prio = 1000
freckle_repos = []
# TODO: pre-fill with adapter-defaults?
for freckle in default_freckle_urls:
repo = FreckleRepo(freckle, target_folder=parent_command_vars["target_folder"], target_name=parent_command_vars["target_folder_name"], include=parent_command_vars["include"], exclude=parent_command_vars["exclude"], non_recursive=parent_command_vars["non_recursive"], priority=prio, default_vars={}, overlay_vars={"freckle": default_extra_vars})
prio = prio + 100
freckle_repos.append(repo)
details = FreckleDetails(freckle_repos, profiles_to_run=None)
freckle_details.append(details)
else:
multi_freckle_repos = OrderedDict()
det_prio = 10000
for p in args[0]:
pn = p["name"]
# if pn in profiles.keys():
# raise Exception("Profile '{}' specified twice. I don't think that makes sense. Exiting...".format(pn))
metadata = {}
metadata = {}
metadata["metadata"] = p["adapter_metadata"]
metadata["details"] = p["adapter_details"]
pvars_adapter_defaults = p["default_vars"]
pvars_extra_vars = p["extra_vars"]
pvars_user_input = p["user_input"]
pvars_profile_extra_vars = pvars_user_input.pop("profile_extra_vars", ())
freckle_default_vars = OrderedDict()
for ev in pvars_extra_vars:
frkl.dict_merge(freckle_default_vars, ev, copy_dct=False)
pvars = OrderedDict()
for ev in pvars_profile_extra_vars:
frkl.dict_merge(pvars, ev, copy_dct=False)
frkl.dict_merge(pvars, pvars_user_input, copy_dct=False)
freckles = list(pvars.pop("freckle", []))
include = list(set(pvars.pop("include", [])))
exclude = list(set(pvars.pop("exclude", [])))
target_folder = pvars.pop("target_folder", None)
target_name = pvars.pop("target_name", None)
# ask_become_pass = pvars.pop("ask_become_pass", None)
# if ask_become_pass is None:
# ask_become_pass = default_ask_become_pass
non_recursive = pvars.pop("non_recursive", False)
if non_recursive is None:
non_recursive = default_non_recursive
log.debug("Merged vars for profile: freckle".format(pn))
log.debug(readable_json(freckle_default_vars, indent=2))
log.debug("Merged vars for profile: {}".format(pn))
log.debug(readable_json(pvars, indent=2))
all_freckles_for_this_profile = freckles + default_freckle_urls
if len(all_freckles_for_this_profile) > 1 and target_name is not None:
raise Exception("Can't use 'target_name' if more than one folders are specified")
prio = 1000
freckle_repos = []
for freckle in all_freckles_for_this_profile:
repo = FreckleRepo(freckle, target_folder=target_folder, target_name=target_name, include=include, exclude=exclude, non_recursive=non_recursive, priority=prio, default_vars={pn: pvars_adapter_defaults}, overlay_vars={pn: pvars, "freckle": freckle_default_vars})
prio = prio + 100
freckle_repos.append(repo)
details = FreckleDetails(freckle_repos, profiles_to_run=pn, detail_priority=det_prio)
freckle_details.append(details)
det_prio = det_prio + 1000
if default_password is None:
default_password = "******"
if default_password == "ask":
password = click.prompt("Please enter sudo password for this run", hide_input=True)
click.echo()
default_password = False
# TODO: check password valid
elif default_password == "ansible":
default_password = True
password = None
elif default_password == "no":
default_password = False
password = None
else:
raise click.ClickException("Can't process password: {}".format(default_password))
try:
f = Freckelize(freckle_details, ask_become_pass=default_password, password=password)
f.execute(hosts=hosts, no_run=no_run, output_format=default_output_format)
except (Exception) as e:
raise click.ClickException(str(e))
sys.exit(0)
def create_and_run_nsbl_runner(task_config, task_metadata={}, output_format="default", ask_become_pass=False, password=None,
pre_run_callback=None, no_run=False, additional_roles=[], config=None, run_box_basics=False, additional_repo_paths=[], hosts_list=["localhost"]):
if run_box_basics:
result = execute_run_box_basics(output_format, ask_become_pass, password)
if result["return_code"] > 0:
return result
if not config:
config = DEFAULT_FRECKLES_CONFIG
if additional_repo_paths:
if config.use_freckle_as_repo:
config.add_repo(additional_repo_paths)
config_trusted_repos = config.trusted_repos
local_role_repos = []
for repo_name in config_trusted_repos:
repo = DEFAULT_ROLE_REPOS.get(repo_name)
if not repo:
if not os.path.exists(repo_name):
repo_details = nsbl_tasks.expand_string_to_git_details(repo_name, DEFAULT_ABBREVIATIONS)
repo_url = repo_details["url"]
repo_branch = repo_details.get('branch', None)
relative_repo_path = nsbl_tasks.calculate_local_repo_path(repo_url, repo_branch)
repo_path = os.path.join(DEFAULT_LOCAL_REPO_PATH_BASE, relative_repo_path)
else:
local_role_repos.append(repo_name)
else:
repo_path = repo["path"]
local_role_repos.append(repo_path)
role_repos = nsbl_defaults.calculate_role_repos(local_role_repos)
task_descs = config.task_descs
global_vars = {}
for tc in task_config:
v = tc.get("vars", {})
if v:
dict_merge(global_vars, v, copy_dct=False)
nsbl_obj = nsbl.Nsbl.create(task_config, role_repos, task_descs, wrap_into_hosts=hosts_list, pre_chain=[],
additional_roles=additional_roles)
runner = nsbl.NsblRunner(nsbl_obj)
run_target = os.path.expanduser(DEFAULT_RUN_LOCATION)
ansible_verbose = ""
stdout_callback = "nsbl_internal"
ignore_task_strings = []
display_sub_tasks = True
display_skipped_tasks = False
if output_format == "verbose":
stdout_callback = "default"
ansible_verbose = "-vvvv"
elif output_format == "ansible":
stdout_callback = "default"
elif output_format == "skippy":
stdout_callback = "skippy"
elif output_format == "default_full":
stdout_callback = "nsbl_internal"
display_skipped_tasks = True
elif output_format == "default":
ignore_task_strings = DEFAULT_IGNORE_STRINGS
stdout_callback = "nsbl_internal"
elif output_format == "yaml":
stdout_callback = "yaml"
else:
raise Exception("Invalid output format: {}".format(output_format))
force = True
extra_paths = "$HOME/.local/share/inaugurate/virtualenvs/freckles/bin:$HOME/.local/share/inaugurate/conda/envs/freckles/bin"
return runner.run(run_target, global_vars=global_vars, force=force, ansible_verbose=ansible_verbose, ask_become_pass=ask_become_pass, password=password,
extra_plugins=EXTRA_FRECKLES_PLUGINS, callback=stdout_callback, add_timestamp_to_env=True,
add_symlink_to_env=DEFAULT_RUN_SYMLINK_LOCATION, no_run=no_run,
display_sub_tasks=display_sub_tasks, display_skipped_tasks=display_skipped_tasks,
display_ignore_tasks=ignore_task_strings, pre_run_callback=pre_run_callback, extra_paths=extra_paths)
def parse_tasks_dictlet(content,
current_vars,
tasks_keyword=FX_TASKS_KEY_NAME,
vars_keyword=None,
delimiter_profile=JINJA_DELIMITER_PROFILES["luci"]):
"""Process a frecklecutable line-by-line.
The main purpose of this is to extract a task list and vars, as well as
potential other metadata ('defaults', '__freckle__').
The parsing is a bit convoluted, as the metadata as well as 'vars' can be in a commented section,
so 'valid' ansible tasks lists can be used here.
The 'vars' and 'tasks' keys need to be at the lasts ones (in this order, preferrably).
The 'vars' key can either be in a commented section, or not.
If using comments, all the keys in commented section need to have the same comment prefix.
"""
log.debug("Processing: {}".format(content))
# now, I know this isn't really the most
# optimal way of doing this,
# but I don't really care that much about execution speed yet,
# plus I really want to be able to use variables used in previous
# lines of the content
last_whitespaces = 0
current_lines = ""
temp_vars = copy.deepcopy(current_vars)
meta_started = False
tasks_started = False
tasks_finished = False
vars_started = False
vars_finished = False
vars_meta_started = False
vars_meta_finished = False
tasks_string = ""
vars_string = ""
for line in content:
# print("LINE: "+line)
if not tasks_started and not vars_started:
if line.startswith("{}:".format(tasks_keyword)):
tasks_started = True
continue
elif vars_keyword and line.startswith("{}:".format(vars_keyword)):
vars_started = True
continue
if not meta_started:
if "defaults:" in line:
ignore_prefix = line.partition("defaults:")[0]
elif "__freckles__:" in line:
ignore_prefix = line.partition("__freckles__:")[0]
elif "doc:" in line:
ignore_prefix = line.partition("doc:")[0]
elif "args:" in line:
ignore_prefix = line.partition("args:")[0]
elif vars_keyword and "{}:".format(vars_keyword) in line:
ignore_prefix = line.partition("vars:")[0]
vars_meta_started = True
meta_started = True
continue
else:
continue
meta_started = True
else:
if ignore_prefix and not line.startswith(
ignore_prefix) and not (ignore_prefix.strip()
and line.startswith(
ignore_prefix.strip())):
meta_started = False
continue
line_new = line[len(ignore_prefix):]
if vars_keyword and line_new.startswith(
"{}:".format(vars_keyword)):
vars_meta_started = True
continue
if vars_meta_started:
vars_string = "{}\n{}".format(vars_string, line_new)
continue
whitespaces = len(line_new) - len(line_new.lstrip(' '))
current_lines = "{}{}\n".format(current_lines, line_new)
if whitespaces <= last_whitespaces:
if delimiter_profile:
temp = replace_string(current_lines, temp_vars,
**delimiter_profile)
else:
temp = current_lines
if not temp.strip():
continue
temp_dict = ordered_load(temp)
temp_vars = frkl.dict_merge(temp_vars,
temp_dict,
copy_dct=False)
last_whitespaces = whitespaces
else:
if vars_keyword and line.startswith("{}:".format(vars_keyword)):
if tasks_started:
tasks_finished = True
tasks_started = False
if vars_finished:
raise Exception(
"Can't have two segments starting with '{}' in frecklecutable."
.format(self.vars_keyword))
vars_started = True
elif line.startswith("{}:".format(tasks_keyword)):
if vars_started:
vars_finished = True
vars_started = False
if tasks_finished:
raise Exception(
"Can't have two segments starting with '{}' in frecklecutable"
.format(self.tasks_keyword))
tasks_started = True
else:
if vars_started:
vars_string = "{}\n{}".format(vars_string, line)
elif tasks_started:
tasks_string = "{}\n{}".format(tasks_string, line)
else:
raise Exception(
"Internal error in frecklecutable reader. Please report an issue."
)
if current_lines:
if delimiter_profile:
temp = replace_string(
current_lines,
temp_vars,
additional_jinja_extensions=freckles_jinja_extensions,
**delimiter_profile)
else:
temp = current_lines
if temp.strip():
temp_dict = ordered_load(temp)
temp_vars = frkl.dict_merge(temp_vars, temp_dict, copy_dct=False)
frkl.dict_merge(current_vars, temp_vars, copy_dct=False)
current_vars[tasks_keyword] = tasks_string
current_vars[vars_keyword] = vars_string
log.debug("Vars after processing: {}".format(current_vars))
return current_vars
def add_default_vars(self, vars_dict):
frkl.dict_merge(self.default_vars, vars_dict, copy_dct=False)
def add_overlay_vars(self, vars_dict):
frkl.dict_merge(self.overlay_vars, vars_dict, copy_dct=False)
def render(self,
env_dir,
extra_plugins=None,
extract_vars=True,
force=False,
ask_become_pass="******",
ansible_args="",
callback='default',
force_update_roles=False,
add_timestamp_to_env=False,
add_symlink_to_env=False):
"""Creates the ansible environment in the folder provided.
Args:
env_dir (str): the folder where the environment should be created
extra_plugins (str): a path to a repository of extra ansible plugins, if necessary
extract_vars (bool): whether to extract a hostvars and groupvars directory for the inventory (True), or render a dynamic inventory script for the environment (default, True) -- Not supported at the moment
force (bool): overwrite environment if already present at the specified location, use with caution because this might delete an important folder if you get the 'target' dir wrong
ask_become_pass (str): whether to include the '--ask-become-pass' arg to the ansible-playbook call, options: 'auto', 'true', 'false'
ansible_verbose (str): parameters to give to ansible-playbook (like: "-vvv")
callback (str): name of the callback to use, default: nsbl_internal
force_update_roles (bool): whether to overwrite external roles that were already downloaded
add_timestamp_to_env (bool): whether to add a timestamp to the env_dir -- useful for when this is called from other programs (e.g. freckles)
add_symlink_to_env (bool): whether to add a symlink to the current env from a fixed location (useful to archive all runs/logs)
"""
if isinstance(ask_become_pass, bool):
ask_become_pass = str(ask_become_pass)
if not isinstance(ask_become_pass,
string_types) or ask_become_pass.lower() not in [
"true", "false", "auto"
]:
raise NsblException(
"Can't parse 'ask_become_pass' var: '{}'".format(
ask_become_pass))
if ask_become_pass == "auto":
ask_become_pass = self.use_become
else:
ask_become_pass = ask_become_pass.lower() in (['true', 'yes'])
env_dir = os.path.expanduser(env_dir)
if add_timestamp_to_env:
start_date = datetime.now()
date_string = start_date.strftime('%y%m%d_%H_%M_%S')
dirname, basename = os.path.split(env_dir)
env_dir = os.path.join(dirname,
"{}_{}".format(basename, date_string))
result = {}
result['env_dir'] = env_dir
if os.path.exists(env_dir) and force:
shutil.rmtree(env_dir)
inventory_dir = os.path.join(env_dir, "inventory")
result["inventory_dir"] = inventory_dir
if extract_vars:
inv_target = "../inventory/hosts"
else:
inv_target = "../inventory/inventory"
result["extract_vars"] = extract_vars
playbook_dir = os.path.join(env_dir, "plays")
result["playbook_dir"] = playbook_dir
roles_base_dir = os.path.join(env_dir, "roles")
result["roles_base_dir"] = playbook_dir
# ask_sudo = "--ask-become-pass"
if ask_become_pass:
if can_passwordless_sudo():
ask_sudo = ""
else:
ask_sudo = "--ask-become-pass"
else:
ask_sudo = ""
all_plays_name = "all_plays.yml"
result["default_playbook_name"] = all_plays_name
ansible_playbook_args = ansible_args
result["ansible_playbook_cli_args"] = ansible_playbook_args
result["run_playbooks_script"] = os.path.join(env_dir,
"run_all_plays.sh")
try:
import ara
ara_base = os.path.dirname(ara.__file__)
except:
ara_base = None
pass
if ara_base:
callback_plugins_list = "callback_plugins:{}/plugins/callbacks".format(
ara_base)
action_plugins_list = "action_plugins:{}/plugins/actions".format(
ara_base)
library_plugins_list = "library:{}/plugins/modules".format(
ara_base)
else:
callback_plugins_list = "callback_plugins"
action_plugins_list = "action_plugins"
library_plugins_list = "library"
cookiecutter_details = {
"inventory": inv_target,
"env_dir": env_dir,
"playbook_dir": playbook_dir,
"ansible_playbook_args": ansible_playbook_args,
"library_path": library_plugins_list,
"action_plugins_path": action_plugins_list,
"extra_script_commands": "",
"ask_sudo": ask_sudo,
"playbook": all_plays_name,
"callback_plugins": callback_plugins_list,
"callback_plugin_name": callback,
"callback_whitelist": "default_to_file"
}
log.debug("Creating build environment from template...")
log.debug(
"Using cookiecutter details: {}".format(cookiecutter_details))
template_path = os.path.join(os.path.dirname(__file__), "external",
"cookiecutter-ansible-environment")
cookiecutter(template_path,
extra_context=cookiecutter_details,
no_input=True)
if add_symlink_to_env:
link_path = os.path.expanduser(add_symlink_to_env)
if os.path.exists(link_path) and force:
os.unlink(link_path)
link_parent = os.path.abspath(os.path.join(link_path, os.pardir))
try:
os.makedirs(link_parent)
except:
pass
os.symlink(env_dir, link_path)
# write inventory
if extract_vars:
self.inventory.extract_vars(inventory_dir)
self.inventory.write_inventory_file_or_script(
inventory_dir, extract_vars=extract_vars)
# write roles
all_playbooks = []
ext_roles = False
roles_to_copy = {}
task_details = []
for play, tasks in self.plays.items():
task_details.append(str(tasks))
playbook = tasks.render_playbook(playbook_dir)
all_playbooks.append(playbook)
tasks.render_roles(roles_base_dir)
if tasks.roles_to_copy:
dict_merge(roles_to_copy, tasks.roles_to_copy, copy_dct=False)
if tasks.ext_roles:
ext_roles = True
result["task_details"] = task_details
jinja_env = Environment(loader=PackageLoader('nsbl', 'templates'))
template = jinja_env.get_template('play.yml')
output_text = template.render(playbooks=all_playbooks)
all_plays_file = os.path.join(env_dir, "plays", all_plays_name)
result["all_plays_file"] = all_plays_file
with open(all_plays_file, "w") as text_file:
text_file.write(output_text)
# copy extra_plugins
library_path = os.path.join(os.path.dirname(__file__), "external",
"extra_plugins", "library")
action_plugins_path = os.path.join(os.path.dirname(__file__),
"external", "extra_plugins",
"action_plugins")
callback_plugins_path = os.path.join(os.path.dirname(__file__),
"external", "extra_plugins",
"callback_plugins")
result["library_path"] = library_path
result["callback_plugins_path"] = callback_plugins_path
result["action_plugins_path"] = action_plugins_path
target_dir = playbook_dir
if extra_plugins:
dirs = [
o for o in os.listdir(extra_plugins)
if os.path.isdir(os.path.join(extra_plugins, o))
]
for d in dirs:
shutil.copytree(os.path.join(extra_plugins, d),
os.path.join(target_dir, d))
if ext_roles:
# download external roles
click.echo("\nDownloading external roles...")
role_requirement_file = os.path.join(env_dir, "roles",
"roles_requirements.yml")
if not os.path.exists(ANSIBLE_ROLE_CACHE_DIR):
os.makedirs(ANSIBLE_ROLE_CACHE_DIR)
command = [
"ansible-galaxy", "install", "-r", role_requirement_file, "-p",
ANSIBLE_ROLE_CACHE_DIR
]
if force_update_roles:
command.append("--force")
log.debug("Downloading and installing external roles...")
my_env = os.environ.copy()
my_env["PATH"] = "{}:{}:{}".format(
os.path.expanduser("~/.local/bin"),
os.path.expanduser("~/.local/inaugurate/bin"), my_env["PATH"])
res = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
env=my_env)
for line in iter(res.stdout.readline, ""):
if "already installed" not in line and "--force to change" not in line:
# log.debug("Installing role: {}".format(line.encode('utf8')))
click.echo(" {}".format(line.encode('utf8')), nl=False)
if roles_to_copy.get("internal", {}):
for src, target in roles_to_copy["internal"].items():
log.debug("Coping internal role: {} -> {}".format(src, target))
shutil.copytree(src, target)
if roles_to_copy.get("external", {}):
for src, target in roles_to_copy["external"].items():
log.debug("Coping external role: {} -> {}".format(src, target))
shutil.copytree(src, target)
return result
def create_freckle_descs(repos, config=None):
"""Augments freckle urls provided by the user via cli (if necessary).
"""
for temp_url, metadata in repos.items():
target = metadata["target_folder"]
source_delete = False
if target != DEFAULT_FRECKLE_TARGET_MARKER and not target.startswith(
"~") and not target.startswith("/"):
raise Exception(
"Relative path not supported for target option, please use an absolute one"
)
if target == DEFAULT_FRECKLE_TARGET_MARKER or target.startswith(
"~") or target.startswith("/home"):
target_become = False
else:
target_become = True
if temp_url.startswith(BLUEPRINT_URL_PREFIX):
blueprint_name = ":".join(temp_url.split(":")[1:])
blueprints = get_available_blueprints(config)
match = blueprints.get(blueprint_name, False)
if not match:
raise Exception(
"No blueprint with name '{}' available.".format(
blueprint_name))
if target == DEFAULT_FRECKLE_TARGET_MARKER:
raise Exception(
"No target directory specified when using blueprint.")
cookiecutter_file = os.path.join(match, "cookiecutter.json")
if os.path.exists(cookiecutter_file):
click.secho(
"\nFound interactive blueprint, please enter approriate values below:\n",
bold=True)
temp_path = tempfile.mkdtemp(prefix='frkl.')
cookiecutter(match, output_dir=temp_path)
subdirs = [
os.path.join(temp_path, f) for f in listdir(temp_path)
if isdir(join(temp_path, f))
]
if len(subdirs) != 1:
raise Exception(
"More than one directories created by interactive template '{}'. Can't deal with that."
.format(match))
url = subdirs[0]
source_delete = True
else:
url = match
else:
url = temp_url
if os.path.exists(os.path.expanduser(url)):
# assuming archive
if os.path.isfile(os.path.expanduser(url)):
# TODO: check whether host is local
repo_desc = {"type": "local_archive"}
repo_desc["remote_url"] = os.path.abspath(
os.path.expanduser(url))
repo_desc["checkout_become"] = target_become
#repo_desc["local_name"] = os.path.basename(url).split('.')[0]
repo_desc["source_delete"] = False
if target == DEFAULT_FRECKLE_TARGET_MARKER:
raise Exception(
"No target directory specified when using archive file."
)
repo_desc["local_parent"] = target
repo_desc["checkout_skip"] = False
else:
# TODO: check whether host is local
repo_desc = {"type": "local"}
repo_desc["remote_url"] = os.path.abspath(
os.path.expanduser(url))
repo_desc["checkout_become"] = target_become
repo_desc["local_name"] = os.path.basename(
repo_desc["remote_url"])
repo_desc["source_delete"] = source_delete
if target == DEFAULT_FRECKLE_TARGET_MARKER:
repo_desc["checkout_skip"] = True
repo_desc["local_parent"] = os.path.dirname(
repo_desc["remote_url"])
else:
repo_desc["local_parent"] = target
repo_desc["checkout_skip"] = False
elif url.endswith(".git"):
repo_desc = {"type": "git"}
repo_desc["remote_url"] = url
repo_desc["checkout_become"] = target_become
repo_desc["local_name"] = os.path.basename(
repo_desc["remote_url"])[:-4]
repo_desc["checkout_skip"] = False
if target == DEFAULT_FRECKLE_TARGET_MARKER:
repo_desc["local_parent"] = "~/freckles"
else:
repo_desc["local_parent"] = target
elif url.startswith("http://") or url.startswith("https://"):
# TODO: check whether host is local
repo_desc = {"type": "remote_archive"}
repo_desc["remote_url"] = url
repo_desc["checkout_become"] = target_become
#repo_desc["local_name"] = os.path.basename(url).split('.')[0]
repo_desc["source_delete"] = False
if target == DEFAULT_FRECKLE_TARGET_MARKER:
raise Exception(
"No target directory specified when using archive file.")
repo_desc["local_parent"] = target
repo_desc["checkout_skip"] = False
else:
raise Exception(
"freckle url format unknown, don't know how to handle that: {}"
.format(url))
repo_desc["blueprint"] = temp_url.startswith(BLUEPRINT_URL_PREFIX)
frkl.dict_merge(metadata, repo_desc, copy_dct=False)
def __init__(self,
name,
tasks,
vars,
tasks_format=None,
external_task_list_map={},
additional_roles=[]):
self.name = name
self.tasks = tasks
if isinstance(tasks, string_types):
self.tasks = yaml.safe_load(tasks)
self.tasks_string = tasks
elif isinstance(tasks, (list, tuple)):
self.tasks = tasks
self.tasks_string = yaml.safe_dump(tasks, default_flow_style=False)
else:
raise Exception("Invalid type for tasks list: {}".format(
type(tasks)))
self.vars = vars
self.metadata = {} # not used at the moment
if tasks_format == None:
log.debug("Trying to guess task-list format...")
tasks_format = get_task_list_format(self.tasks)
if tasks_format == None:
log.info(
"Could not determine task list format for sure, falling back to '{}'."
.format(DEFAULT_FRECKLECUTALBE_TASK_LIST_FORMAT))
tasks_format = DEFAULT_FRECKLECUTALBE_TASK_LIST_FORMAT
self.tasks_format = tasks_format
if self.tasks_format not in ["freckles", "ansible"]:
raise Exception("Invalid task-list format: {}".format(
self.tasks_format))
self.external_task_list_map = external_task_list_map
# getting ansible roles, this is not necessary for the 'freckles' configuration format
self.additional_roles = additional_roles
self.task_list_aliases = {}
for name, details in self.external_task_list_map.items():
self.task_list_aliases[name] = details["play_target"]
# generating rendered tasks depending on task-list format
if self.tasks_format == "ansible":
relative_target_file = os.path.join(
"{{ playbook_dir }}", "..", "task_lists",
"frecklecutable_default_tasks.yml")
self.final_tasks = [{
"meta": {
"name": "include_tasks",
"task-desc": "[including tasks]",
"var-keys": ["free_form"],
},
"vars": {
"free_form": relative_target_file
}
}]
else:
self.final_tasks = self.tasks
self.all_vars = frkl.dict_merge(vars,
self.task_list_aliases,
copy_dct=True)
self.task_config = [{"tasks": self.final_tasks, "vars": self.all_vars}]
def freckles_process(self, command_name, default_vars, extra_vars,
user_input, metadata, dictlet_details, config,
parent_params, command_var_spec):
all_vars = OrderedDict()
frkl.dict_merge(all_vars, default_vars, copy_dct=False)
for ev in extra_vars:
frkl.dict_merge(all_vars, ev, copy_dct=False)
frkl.dict_merge(all_vars, user_input, copy_dct=False)
hosts = parent_params.get("hosts", ["localhost"])
output_format = parent_params.get("output", "default")
password_type = parent_params.get("password", None)
no_run = parent_params.get("no_run", False)
tasks_string = metadata.get(FX_TASKS_KEY_NAME, "")
vars_string = metadata.get(FX_VARS_KEY_NAME, "")
replaced_vars = replace_string(
vars_string,
all_vars,
additional_jinja_extensions=freckles_jinja_extensions,
**JINJA_DELIMITER_PROFILES["luci"])
try:
vars_dictlet = yaml.safe_load(replaced_vars)
except (Exception) as e:
raise Exception("Can't parse vars: {}".format(e))
if vars_dictlet:
temp_new_all_vars = frkl.dict_merge(all_vars,
vars_dictlet,
copy_dct=True)
else:
temp_new_all_vars = all_vars
replaced_tasks = replace_string(
tasks_string,
temp_new_all_vars,
additional_jinja_extensions=freckles_jinja_extensions,
**JINJA_DELIMITER_PROFILES["luci"])
try:
tasks_list_temp = ordered_load(replaced_tasks)
except (Exception) as e:
raise click.ClickException(
"Could not parse frecklecutable '{}': {}".format(
command_name, e))
extra_task_lists_map = process_extra_task_lists(
metadata, dictlet_details["path"])
# check for hardcoded task_list_format:
task_list_format = metadata.get("__freckles__",
{}).get("task_list_format", None)
additional_roles = metadata.get("__freckles__", {}).get("roles", [])
result_vars = {}
for name, details in command_var_spec.items():
if name in temp_new_all_vars and details.get("is_var",
False) == True:
result_vars[name] = temp_new_all_vars[name]
f = Frecklecutable(command_name,
replaced_tasks,
result_vars,
tasks_format=task_list_format,
external_task_list_map=extra_task_lists_map,
additional_roles=additional_roles)
# placeholder, for maybe later
task_metadata = {}
if password_type is None:
password_type = "no"
if password_type == "ask":
password = click.prompt("Please enter sudo password for this run",
hide_input=True)
click.echo()
password_type = False
# TODO: check password valid
elif password_type == "ansible":
password_type = True
password = None
elif password_type == "no":
password_type = False
password = None
else:
raise Exception("Can't process password: {}".format(password_type))
run = Frecklecute(f,
config=self.config,
ask_become_pass=password_type,
password=password)
run.execute(hosts=hosts, no_run=no_run, output_format=output_format)