def create_package_list_from_profiles_metadata(
self, freckelize_profiles_metadata, default_pkg_mgr):
"""
Tries to get all packages from all freckle items.
"""
result = []
for profile, folder_list in freckelize_profiles_metadata.items():
for folder_metadata in folder_list:
packages = folder_metadata["folder_vars"].get("install", [])
if not packages:
continue
parent_vars = copy.deepcopy(folder_metadata["vars"])
parent_vars.pop("install", None)
if "pkg_mgr" not in parent_vars.keys():
parent_vars["pkg_mgr"] = default_pkg_mgr
pkg_config = {"vars": parent_vars, "packages": packages}
chain = [frkl.FrklProcessor(DEFAULT_PACKAGE_FORMAT)]
frkl_obj = frkl.Frkl(pkg_config, chain)
pkgs = frkl_obj.process()
result = result + pkgs
# TODO: remove duplicates?
#return sorted(result, key=lambda k: k.get("vars", {}).get("name", "zzz"))
return result
def calculate_task_descs(task_descs, role_repos=[], add_upper_case_versions=True):
"""Utility method to calculate which task descriptions to use.
Task descriptions are yaml files that translate task-names in a task config
into roles or ansible tasks, optionally with extra default parameters.
If additional role_repos are provided, we will check whether each of them
contains a file with the value of TASK_DESC_DEFAULT_FILENAME. If so, those
will be added to the beginning of the resulting list.
Args:
task_descs (list): a string or list of strings of local files
role_repos (list): a list of role repos (see 'calculate_role_repos' method)
add_upper_case_versions (bool): if true, will add an upper-case version of every task desc that includes a meta/become = true entry
Returns:
list: a list of dicts of all task description configs to be used
"""
if not task_descs:
task_descs = []
if isinstance(task_descs, string_types):
task_descs = [task_descs]
elif not isinstance(task_descs, (list, tuple)):
raise Exception("task_descs needs to be string or list: '{}'".format(task_descs))
if role_repos:
repo_task_descs = []
for repo in role_repos:
task_desc_file = os.path.join(os.path.expanduser(repo), TASK_DESC_DEFAULT_FILENAME)
if os.path.exists(task_desc_file):
repo_task_descs.append(task_desc_file)
task_descs = repo_task_descs + task_descs
# TODO: check whether paths exist
frkl_format = generate_nsbl_tasks_format([])
task_desk_frkl = frkl.Frkl(task_descs, [frkl.UrlAbbrevProcessor(),
frkl.EnsureUrlProcessor(),
frkl.EnsurePythonObjectProcessor(),
frkl.FrklProcessor(frkl_format)])
processed_task_descs = task_desk_frkl.process()
if add_upper_case_versions:
result = []
for task in processed_task_descs:
result.append(task)
task_become = copy.deepcopy(task)
task_become[TASKS_META_KEY][TASK_META_NAME_KEY] = task[TASKS_META_KEY][TASK_META_NAME_KEY].upper()
task_become[TASKS_META_KEY][TASK_BECOME_KEY] = True
result.append(task_become)
return result
else:
return processed_task_descs
def parse_config_file(path):
chain = [
frkl.EnsureUrlProcessor(),
frkl.EnsurePythonObjectProcessor(),
frkl.FrklProcessor(DEFAULT_PROFILE_VAR_FORMAT)
]
frkl_obj = frkl.Frkl(path, chain)
# mdrc_init = {"append_keys": "vars/packages"}
# frkl_callback = frkl.MergeDictResultCallback(mdrc_init)
frkl_callback = frkl.MergeResultCallback()
configs = frkl_obj.process(frkl_callback)
repos = DEFAULT_ROLE_REPOS
urls = DEFAULT_TRUSTED_URLS
aliases = []
for c in configs:
if c.get("profile", {}).get("name", None) == "config":
temp = c.get("vars", {}).get("trusted-repos", [])
for r in temp:
if os.path.exists(os.path.expanduser(r)):
expanded = os.path.abspath(os.path.expanduser(r))
else:
expanded = r
if expanded not in repos:
repos.append(expanded)
temp = c.get("vars", {}).get("trusted-urls", [])
for u in temp:
if u not in urls:
urls.append(u)
temp = c.get("vars", {}).get("task-aliases", [])
for a in temp:
if a not in aliases:
aliases.append(a)
return {
'trusted-repos': repos,
"trusted-urls": urls,
"task-aliases": aliases
}
def assemble_package_list(self,
vars_dict,
packages_vars_name="install",
default_pkg_mgr="auto"):
packages = vars_dict.get(packages_vars_name, [])
if not packages:
return []
parent_vars = copy.deepcopy(vars_dict)
parent_vars.pop(packages_vars_name, None)
if "pkg_mgr" not in parent_vars.keys():
parent_vars["pkg_mgr"] = default_pkg_mgr
pkg_config = {"vars": parent_vars, "packages": packages}
chain = [frkl.FrklProcessor(DEFAULT_PACKAGE_FORMAT)]
frkl_obj = frkl.Frkl(pkg_config, chain)
pkgs = frkl_obj.process()
return pkgs
def frklize(self, freckle_vars, var_to_extract, frkl_format):
result = []
for folder, folder_metadata in freckles_profile_metadata.items():
var_list = folder_metadata.get("vars", [])
for metadata in var_list:
parent_vars = copy.deepcopy(metadata.get("vars", {}))
parent_vars.pop(var_to_extract, None)
packages = metadata.get("vars", {}).get(var_to_extract, [])
pkg_config = {"vars": parent_vars, var_to_extract: packages}
chain = [frkl.FrklProcessor(frkl_format)]
frkl_obj = frkl.Frkl(pkg_config, chain)
pkgs = frkl_obj.process()
result = result + pkgs
return sorted(result,
key=lambda k: k.get("vars", {}).get("name", "zzz"))
ENV_TYPE_HOST = 'host'
ENV_TYPE_GROUP = 'group'
DEFAULT_ENV_TYPE = ENV_TYPE_GROUP
# tasks that emit 'nsbl'-specific events: nsbl_item_started, nsbl_item_ok, nsbl_item_failed
NSBLIZED_TASKS = ["install"]
DEFAULT_NSBL_TASKS_BOOTSTRAP_FORMAT = {
frkl.CHILD_MARKER_NAME: TASKS_KEY,
frkl.DEFAULT_LEAF_NAME: TASKS_META_KEY,
frkl.DEFAULT_LEAFKEY_NAME: TASK_META_NAME_KEY,
frkl.KEY_MOVE_MAP_NAME: {'*': (VARS_KEY, 'default')},
"use_context": True
}
DEFAULT_NSBL_TASKS_BOOTSTRAP_CHAIN = [
frkl.FrklProcessor(DEFAULT_NSBL_TASKS_BOOTSTRAP_FORMAT)
]
# bootstrap frkl processor chain for creating the inventory hosts/groups lists
NSBL_INVENTORY_BOOTSTRAP_FORMAT = {
frkl.CHILD_MARKER_NAME: ENVS_KEY,
frkl.DEFAULT_LEAF_NAME: ENV_META_KEY,
frkl.DEFAULT_LEAFKEY_NAME: ENV_NAME_KEY,
frkl.OTHER_KEYS_NAME: [VARS_KEY, TASKS_KEY],
frkl.KEY_MOVE_MAP_NAME: VARS_KEY
}
# bootstrap chain used for creating the inventory
NSBL_INVENTORY_BOOTSTRAP_CHAIN = [
frkl.UrlAbbrevProcessor(), frkl.EnsureUrlProcessor(), frkl.EnsurePythonObjectProcessor(),
frkl.FrklProcessor(NSBL_INVENTORY_BOOTSTRAP_FORMAT)]
def run(self, tmp=None, task_vars=None):
''' handler for template operations '''
self.nsbl_env = os.environ.get("NSBL_ENVIRONMENT", False) == "true"
if task_vars is None:
task_vars = dict()
result = super(ActionModule, self).run(tmp, task_vars)
format = {
"child_marker": "packages",
"default_leaf": "vars",
"default_leaf_key": "name",
"key_move_map": {
'*': "vars"
}
}
chain = [frkl.FrklProcessor(format)]
frkl_obj = frkl.Frkl(self._task.args["packages"], chain)
package = frkl_obj.process()
if len(package) == 0:
raise Exception("No packages provided for package: {}".format(
self._task.args["packages"]))
if len(package) != 1:
raise Exception(
"For some reason more than one package provided, this shouldn't happen: {}"
.format(package))
package = package[0]
if "pkg_mgr" not in package[VARS_KEY].keys():
pkg_mgr = self._task.args.get('pkg_mgr', 'auto')
else:
pkg_mgr = package[VARS_KEY]["pkg_mgr"]
if pkg_mgr == 'auto':
try:
if self._task.delegate_to:
pkg_mgr = self._templar.template(
"{{hostvars['%s']['ansible_facts']['ansible_pkg_mgr']}}"
% self._task.delegate_to)
else:
pkg_mgr = self._templar.template(
'{{ansible_facts["ansible_pkg_mgr"]}}')
except Exception as e:
pass # could not get it from template!
auto = pkg_mgr == 'auto'
facts = self._execute_module(module_name='setup',
module_args=dict(gather_subset='!all'),
task_vars=task_vars)
if auto:
pkg_mgr = facts['ansible_facts'].get('ansible_pkg_mgr', None)
os_family = facts['ansible_facts'].get('ansible_os_family', None)
distribution = facts['ansible_facts'].get('ansible_distribution', None)
distribution_major_version = facts['ansible_facts'].get(
'ansible_distribution_major_version', None)
distribution_version = facts['ansible_facts'].get(
'ansible_distribution_version', None)
distribution_release = facts['ansible_facts'].get(
'ansible_distribution_release', None)
# figure out actual package name
if distribution_version:
full_version_string = "{}-{}".format(distribution,
distribution_version).lower()
else:
full_version_string = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
if distribution_release:
full_release_string = "{}-{}".format(distribution,
distribution_release).lower()
else:
full_release_string = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
if distribution_major_version:
distribution_major_string = "{}-{}".format(
distribution, distribution_major_version).lower()
else:
distribution_major_string = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
distribution_string = distribution.lower()
os_string = os_family.lower()
if pkg_mgr == 'unknown' and os_family == "Darwin":
pkg_mgr = "homebrew"
if pkg_mgr in ['auto', 'unknown']:
result['failed'] = True
result[
'msg'] = 'Could not detect which package manager to use. Try gathering facts or setting the "use" option.'
return result
if pkg_mgr not in self._shared_loader_obj.module_loader:
result['failed'] = True
result[
'msg'] = "Could not find an ansible module for package manager '{}'.".format(
pkg_mgr)
return result
# calculate package name, just in case
pkg_dict = CaseInsensitiveDict(package[VARS_KEY].get("pkgs"))
if pkg_mgr.lower() in (name.lower() for name in pkg_dict.keys()):
calculated_package_pkg_mgr = pkg_dict[pkg_mgr.lower()]
elif 'other' in (name.lower() for name in pkg_dict.keys()):
calculated_package_pkg_mgr = pkg_dict['other']
else:
calculated_package_pkg_mgr = None
if full_version_string in (name.lower() for name in pkg_dict.keys()):
calculated_package_platform = pkg_dict[full_version_string]
elif full_release_string in (name.lower() for name in pkg_dict.keys()):
calculated_package_platform = pkg_dict[full_release_string]
elif distribution_major_string in (name.lower()
for name in pkg_dict.keys()):
calculated_package_platform = pkg_dict[distribution_major_string]
elif distribution_string in (name.lower() for name in pkg_dict.keys()):
calculated_package_platform = pkg_dict[distribution_string]
elif os_string in (name.lower() for name in pkg_dict.keys()):
calculated_package_platform = pkg_dict[os_string]
elif 'other' in (name.lower() for name in pkg_dict.keys()):
calculated_package_platform = pkg_dict['other']
else:
calculated_package_platform = None
# if calculated_package_platform in ['ignore', 'omit'] or calculated_package_pkg_mgr in ['ignore', 'omit']:
# result['msg'] = "Ignoring package {}".format(package[VARS_KEY]["name"])
# result['skipped'] = True
# return result
if not auto or not calculated_package_platform:
calculated_package = calculated_package_pkg_mgr
else:
calculated_package = calculated_package_platform
if calculated_package in ['ignore', 'omit']:
result['msg'] = "Ignoring package {}".format(
package[VARS_KEY]["name"])
result['skipped'] = True
return result
module_result = self.execute_package_module(package,
calculated_package, auto,
pkg_mgr, task_vars, result)
if module_result:
result.update(module_result)
return result
def cli(ctx, use_repo, no_run, output, script):
"""'freckles' command line interface"""
ctx.obj = {}
ctx.obj["config"] = DEFAULT_FRECKLES_CONFIG
download_extra_repos(ctx, None, use_repo)
current_command = None
current_command_path = None
chain = [
frkl.UrlAbbrevProcessor(),
frkl.EnsureUrlProcessor(),
frkl.EnsurePythonObjectProcessor(),
frkl.FrklProcessor(DEFAULT_FRECKLES_COMMAND_FORMAT)
]
frkl_obj = frkl.Frkl(script, chain)
commands = frkl_obj.process()
for item in commands:
command = item["command"]
command_name = command["name"]
command_type = command.get("type", None)
if not command_type:
if "freckle" in item.get("vars", {}).keys():
command_type = 'freckelize'
else:
command_type = 'frecklecute'
# ask_become_pass = item.get("vars", {}).pop("ask-become-pass", False)
# if not ask_become_pass:
# ask_become_pass = item.get("vars", {}).pop("--ask-become-pass", False)
# if not ask_become_pass:
# ask_become_pass = item.get("vars", {}).pop("-pw", False)
arguments = create_cli_list(item.get("vars", {}))
if command_type == 'frecklecute':
command_title = command.get(
"title", "frecklecutable: {}".format(command_name))
if os.path.exists(command_name):
command_path = os.path.abspath(command_name)
else:
command_path = None
cli_command = FrecklecuteCommand((command_name, command_path),
config=DEFAULT_FRECKLES_CONFIG,
help=FRECKLECUTE_HELP_TEXT,
epilog=FRECKLECUTE_EPILOG_TEXT)
click.echo("\n# Executing {}...".format(command_title))
c = cli_command.get_command(ctx, command_name)
if not c:
click.echo(
"Could not find frecklecutable '{}'. Exiting...".format(
command_name))
sys.exit(1)
r = c.main(args=arguments,
prog_name=command_title,
standalone_mode=False)
exit_code = r["return_code"]
elif command_type == 'freckelize':
command_title = command.get(
"title", "freckelize adapter '{}'".format(command_name))
cli_command = FreckelizeCommand(DEFAULT_FRECKLES_CONFIG,
chain=True,
help=FRECKELIZE_HELP_TEXT,
epilog=FRECKELIZE_EPILOG_TEXT)
c = cli_command.get_command(ctx, command_name)
if not c:
click.echo("Could not find adapter '{}'. Exiting...".format(
command_name))
sys.exit(1)
parse_cli_result = c.main(args=arguments,
prog_name=command_title,
standalone_mode=False)
if not parse_cli_result.get("vars", {}).get("freckle", False):
msg = " xxx> no value for key 'freckle' specified in freckelize run '{}'".format(
command_name)
click.echo(msg)
vars_msg = " x> " + pprint.pformat(item.get(
"vars", {})).replace("\n", " x> \n")
click.echo(vars_msg)
click.echo("\nExiting...\n")
sys.exit(1)
click.echo("\n# Applying {}...".format(command_title))
result = cli_command.result_callback(*[[parse_cli_result]])
if not result:
raise Exception("No result")
if no_run != True:
exit_code = result[0]["return_code"]
else:
click.echo("Command type '{}' not supported.".format(command_type))
sys.exit(1)
if exit_code > 0 and no_run != True:
if command_type == "frecklecute":
msg = " xxx> 'frecklecutable' '{}' failed. Used vars:\n".format(
command_name)
else:
msg = " xxx> 'freckelize' using adapter '{}' failed. Used vars:\n".format(
command_name)
click.echo(msg)
vars_msg = " x> " + pprint.pformat(item.get(
"vars", {})).replace('\n', '\n x> ')
click.echo(vars_msg)
click.echo("\nExiting...\n")
sys.exit(exit_code)
def read_checkout_metadata(self, folders_metadata):
temp_vars = OrderedDict()
extra_vars = OrderedDict()
folder_metadata_lookup = {}
for metadata in folders_metadata:
repo_id = metadata["parent_repo_id"]
folder = metadata["full_path"]
folder_metadata_lookup.setdefault(repo_id, {})[folder] = metadata
raw_metadata = metadata.pop(METADATA_CONTENT_KEY, False)
if raw_metadata:
# md = yaml.safe_load(raw_metadata)
md = ordered_load(raw_metadata)
if not md:
md = []
if isinstance(md, dict):
md_temp = []
for key, value in md.items():
md_temp.append({key: value})
md = md_temp
# if isinstance(md, (list, tuple)):
# md = {"vars": md}
else:
md = [{"profile": {"name": "freckle"}, "vars": {}}]
temp_vars.setdefault(repo_id, {}).setdefault(folder, []).append(md)
extra_vars_raw = metadata.pop("extra_vars", False)
if extra_vars_raw:
for rel_path, extra_metadata_raw in extra_vars_raw.items():
extra_metadata = ordered_load(extra_metadata_raw)
if not extra_metadata:
# this means there was an empty file. We interprete that as setting a flag to true
extra_metadata = True
#sub_path, filename = os.path.split(rel_path)
tokens = rel_path.split(os.path.sep)
last_token = tokens[-1]
if last_token.startswith("."):
last_token = last_token[1:]
else:
continue
if last_token.endswith(".freckle"):
last_token = last_token[0:-8]
else:
continue
tokens[-1] = last_token
add_key_to_dict(
extra_vars.setdefault(repo_id,
{}).setdefault(folder, {}),
".".join(tokens), extra_metadata)
# extra_vars.setdefault(folder, {}).setdefault(sub_path, {})[filename[1:-8]] = extra_metadata
result = []
for repo_id, folder_map in temp_vars.items():
for freckle_folder, metadata_list in folder_map.items():
chain = [frkl.FrklProcessor(DEFAULT_PROFILE_VAR_FORMAT)]
try:
frkl_obj = frkl.Frkl(metadata_list, chain)
# mdrc_init = {"append_keys": "vars/packages"}
# frkl_callback = frkl.MergeDictResultCallback(mdrc_init)
frkl_callback = frkl.MergeResultCallback()
profile_vars_new = frkl_obj.process(frkl_callback)
item = {}
item["vars"] = profile_vars_new
item["extra_vars"] = extra_vars.get(repo_id, {}).get(
freckle_folder, {})
item["folder_metadata"] = folder_metadata_lookup[repo_id][
freckle_folder]
result.append(item)
except (frkl.FrklConfigException) as e:
raise Exception(
"Can't read freckle metadata file '{}/.freckle': {}".
format(freckle_folder, e.message))
result.sort(key=lambda k: operator.itemgetter(k["folder_metadata"][
"repo_priority"]))
return result