def create_dotfiles_packages(self, profile_vars): result = [] for folder, subfolder_list in profile_vars.items(): for subfolder_metadata in subfolder_list: md = {} md["stow_source"] = subfolder_metadata[ 'freckles_app_dotfile_folder_path'] md["stow_folder_name"] = subfolder_metadata[ 'freckles_app_dotfile_folder_name'] md["name"] = subfolder_metadata[ 'freckles_app_dotfile_folder_name'] md["stow_folder_parent"] = subfolder_metadata[ 'freckles_app_dotfile_parent_path'] parent_details = subfolder_metadata.get( 'freckles_app_dotfile_parent_details', {}) extra_vars = copy.deepcopy( parent_details.get("extra_vars", {}).get(md["name"], {})) package_md = extra_vars.pop("package", None) overlay = frkl.dict_merge(md, extra_vars) if package_md: frkl.dict_merge(overlay, package_md, copy_dct=False) result.append({"vars": overlay}) return result
def get_vars_from_cli_input(input_args, key_map, task_vars, default_vars, args_that_are_vars, value_vars): # exchange arg_name with var name new_args = {} for key, value in key_map.items(): temp = input_args.pop(key.replace('-', '_')) if key not in args_that_are_vars: if isinstance(temp, tuple): temp = list(temp) new_args[value] = temp else: task_vars[value] = temp # replace all matching strings in value_vars for i, var_name in enumerate(value_vars): if var_name == key: value_vars[i] = value # now overimpose the new_args over template_vars new_args = dict_merge(default_vars, new_args) final_vars = {} sub_dict = copy.deepcopy(new_args) # inject subdict (args and envs) in vars for key, template in task_vars.items(): if isinstance(template, string_types): template_var_string = replace_string(template, sub_dict) if template_var_string.startswith('{') and not \ template_var_string.startswith('{{') and not \ template_var_string.startswith('{%'): # if template_var_string is json, load value # (but do not handle {{ ansible-side substitution) try: template_var_new = yaml.safe_load(template_var_string) final_vars[key] = template_var_new except (Exception) as e: raise Exception("Could not convert template '{}': {}".format(template_var_string, e.message)) else: # else push value final_vars[key] = template_var_string else: final_vars[key] = template new_vars = {} for key in value_vars: values = final_vars.pop(key, {}) if values and not isinstance(values, dict): raise Exception("value for '{}' not a dict: {}".format(key, values)) if values: dict_merge(new_vars, values, copy_dct=False) dict_merge(new_vars, final_vars, copy_dct=False) return new_args, new_vars
def flatten_profile_vars(self, profile_folder_list): if not profile_folder_list: return {} result = {} for f in profile_folder_list: frkl.dict_merge(result, f["vars"], copy_dct=False) return result
def process_folder_vars(self, folder_vars, default_vars, overlay_vars, base_vars={}): final_vars = frkl.dict_merge(default_vars, folder_vars, copy_dct=True) if base_vars: frkl.dict_merge(final_vars, base_vars, copy_dct=False) frkl.dict_merge(final_vars, overlay_vars, copy_dct=False) return final_vars
def process_lines_old(self, content, current_vars): log.debug("Processing content: {}".format(content)) # now, I know this isn't really the most # optimal way of doing this, # but I don't really care that much about execution speed yet, # plus I really want to be able to use variables used in previous # lines of the content last_whitespaces = 0 current_lines = "" temp_vars = copy.deepcopy(current_vars) for line in content: if line.strip().startswith("#"): continue whitespaces = len(line) - len(line.lstrip(' ')) current_lines = "{}{}\n".format(current_lines, line) if whitespaces <= last_whitespaces: temp = replace_string(current_lines, temp_vars, **self.delimiter_profile) if not temp.strip(): continue temp_dict = ordered_load(temp) if temp_dict: temp_vars = frkl.dict_merge(temp_vars, temp_dict, copy_dct=False) last_whitespaces = whitespaces if current_lines: temp = replace_string( current_lines, temp_vars, additional_jinja_extensions=freckles_jinja_extensions, **self.delimiter_profile) temp_dict = ordered_load(temp) temp_vars = frkl.dict_merge(temp_vars, temp_dict, copy_dct=False) frkl.dict_merge(current_vars, temp_vars, copy_dct=False) log.debug("Vars after processing:\n{}".format( readable_json(current_vars, indent=2))) return current_vars
def create_folder_packages(self, package_names, extra_vars): result = [] for name in package_names: details = {"name": name} if name in extra_vars.keys(): no_install = extra_vars.get(name, {}).get("no_install", False) if no_install: continue package_md = extra_vars[name].get("package", None) if package_md: frkl.dict_merge(details, package_md, copy_dct=False) result.append({"vars": details}) return result
def process_dictlet(self, metadata, dictlet_details): freckles_meta = metadata.get(FX_FRECKLES_META_KEY_NAME, {}) defaults = metadata.get(FX_DEFAULTS_KEY_NAME, {}) doc = metadata.get(FX_DOC_KEY_NAME, {}) c_vars_dictlet = metadata.get(FX_ARGS_KEY_NAME, {}) c_vars_command = self.command.get_additional_args() # adapter args take precedence if c_vars_command: c_vars = frkl.dict_merge(c_vars_dictlet, c_vars_command, copy_dct=True) else: c_vars = c_vars_dictlet if not isinstance(c_vars, dict): c_vars = convert_args_to_dict(c_vars) params = parse_args_dict(c_vars) @click.command(cls=FrecklesCliFormatter, name=self.name) @click.pass_context def command(ctx, *args, **kwargs): context_repos = freckles_meta.get("context_repos", []) if context_repos: output = ctx.parent.params.get("output", "default") download_repos(context_repos, self.command.get_config(), output) user_input = clean_user_input(kwargs, c_vars) result = self.command.freckles_process( self.name, defaults, self.extra_vars, user_input, metadata, dictlet_details, config=self.command.get_config(), parent_params=self.parent_params, command_var_spec=c_vars) return result command.params = params help_string = metadata.get(FX_DOC_KEY_NAME, {}).get("help", None) if help_string: command.help = help_string help_details = generate_details(metadata, dictlet_details) command.freckles_cli_details = help_details if "short_help" in doc.keys(): command.short_help = doc.get("short_help") if "epilog" in doc.keys(): command.epilog = doc.get("epilog", None) return command
def replace_string(template_string, replacement_dict): use_environment_vars = True if use_environment_vars: sub_dict = copy.deepcopy({"LOCAL_ENV": os.environ}) dict_merge(sub_dict, replacement_dict, copy_dct=False) else: sub_dict = replacement_dict trim_blocks = True block_start_string = '{%::' block_end_string = '::%}' variable_start_string = '{{::' variable_end_string = '::}}' result = Environment(extensions=[freckles_jinja_utils, ansible_extensions.utils], trim_blocks=trim_blocks, block_start_string=block_start_string, block_end_string=block_end_string, variable_start_string=variable_start_string, variable_end_string=variable_end_string).from_string(template_string).render(sub_dict) return result
def get_all_dictlets(self): """Find all freckelize adapters.""" log.debug("Retrieving all dictlets") if self.adapter_cache is None: self.adapter_cache = {} dictlet_names = OrderedDict() all_adapters = OrderedDict() for path in self.paths: if path not in self.path_cache.keys(): adapters = find_freckelize_adapters(path) self.path_cache[path] = adapters frkl.dict_merge(all_adapters, adapters, copy_dct=False) frkl.dict_merge(self.adapter_cache, adapters, copy_dct=False) return all_adapters
def get_all_dictlets(self): """Find all frecklecutables.""" if self.frecklecutable_cache is None: self.frecklecutable_cache = {} all_frecklecutables = OrderedDict() for path in self.paths: if path not in self.path_cache.keys(): commands = OrderedDict() dirs = find_frecklecutable_dirs(path) for f_dir in dirs: fx = find_frecklecutables_in_folder(f_dir) frkl.dict_merge(commands, fx, copy_dct=False) self.path_cache[path] = commands frkl.dict_merge(self.frecklecutable_cache, commands, copy_dct=False) frkl.dict_merge(all_frecklecutables, self.path_cache[path], copy_dct=False) return all_frecklecutables
def convert(self, value, param, ctx): chain = [ UrlAbbrevProcessor(), EnsureUrlProcessor(), EnsurePythonObjectProcessor(), LoadMoreConfigsProcessor()] try: if not isinstance(value, (list, tuple)): value = [value] frkl_obj = Frkl(value, chain) result = frkl_obj.process() if isinstance(result[0], (list, tuple)): result_dict = {} for item in result[0]: dict_merge(result_dict, item, copy_dct=False) return result_dict else: return result[0] except (Exception) as e: self.fail("Can't read vars '{}': {}".format(value, str(e)))
def assemble_freckle_run(*args, **kwargs): try: result = [] no_run = kwargs.get("no_run") hosts = list(kwargs["host"]) if not hosts: hosts = ["localhost"] default_target = kwargs.get("target_folder", None) if not default_target: default_target = DEFAULT_FRECKLE_TARGET_MARKER default_freckle_urls = list(kwargs.get("freckle", [])) default_output_format = kwargs.get("output", "default") default_include = list(kwargs.get("include", [])) default_exclude = list(kwargs.get("exclude", [])) default_ask_become_pass = kwargs.get("ask_become_pass", True) default_extra_vars_raw = list(kwargs.get("extra_vars", [])) default_non_recursive = kwargs.get("non_recursive", False) if not default_non_recursive: default_non_recursive = False default_extra_vars = {} for ev in default_extra_vars_raw: dict_merge(default_extra_vars, ev, copy_dct=False) extra_profile_vars = {} repos = collections.OrderedDict() profiles = [] metadata = {} if not args[0]: # auto-detect metadata["__auto__"] = {} fr = { "target_folder": default_target, "includes": default_include, "excludes": default_exclude, "password": default_ask_become_pass, "non_recursive": default_non_recursive } for f in default_freckle_urls: repos[f] = copy.deepcopy(fr) repos[f]["profiles"] = ["__auto__"] extra_profile_vars = default_extra_vars else: for p in args[0]: pn = p["name"] metadata[pn] = p["metadata"] if pn in profiles: raise Exception( "Profile '{}' specified twice. I don't think that makes sense. Exiting..." .format(pn)) else: profiles.append(pn) pvars = p["vars"] freckles = list(pvars.pop("freckle", [])) include = set(pvars.pop("include", [])) exclude = set(pvars.pop("exclude", [])) target = pvars.pop("target-folder", None) ask_become_pass = pvars.pop("ask_become_pass", "auto") non_recursive = pvars.pop("non_recursive", False) if non_recursive == None: non_recursive = default_non_recursive if ask_become_pass == "auto" and default_ask_become_pass != "auto": ask_become_pass = default_ask_become_pass extra_profile_vars[pn] = copy.deepcopy( default_extra_vars.get(pn, {})) if pvars: for pk, pv in pvars.items(): if pv != None: extra_profile_vars[pn][pk] = pv all_freckles_for_this_profile = list( set(default_freckle_urls + freckles)) for freckle_url in all_freckles_for_this_profile: if target: t = target else: t = default_target for i in default_include: if i not in include: include.add(i) for e in default_exclude: if e not in exclude: exclude.add(e) fr = { "target_folder": t, "includes": list(include), "excludes": list(exclude), "password": ask_become_pass, "non_recursive": non_recursive } repos.setdefault(freckle_url, fr) repos[freckle_url].setdefault("profiles", []).append(pn) if (repos): click.echo("\n# starting ansible run(s)...") temp = execute_freckle_run(repos, profiles, metadata, extra_profile_vars=extra_profile_vars, no_run=no_run, output_format=default_output_format, ask_become_pass=default_ask_become_pass, hosts_list=hosts) result.append(temp) click.echo("") return result except (Exception) as e: message = e.message if not message: if not e.reason: message = "n/a" else: message = "Reason: {}".format(e.reason) click.echo( "\nError assembling configuration.\nMessage: {}\nExiting...". format(message)) sys.exit(1)
def assemble_freckelize_run(*args, **kwargs): no_run = kwargs.get("no_run") hosts = list(kwargs["host"]) if not hosts: hosts = ["localhost"] default_target = kwargs.get("target_folder", None) default_target_name = kwargs.get("target_name", None) default_freckle_urls = list(kwargs.get("freckle", [])) default_output_format = kwargs.get("output", "default") default_include = list(kwargs.get("include", [])) default_exclude = list(kwargs.get("exclude", [])) default_password = kwargs.get("password", None) default_non_recursive = kwargs.get("non_recursive", None) default_extra_vars_list = list(kwargs.get("vars", [])) default_extra_vars = OrderedDict() for ev in default_extra_vars_list: frkl.dict_merge(default_extra_vars, ev, copy_dct=False) parent_command_vars = {} if default_target: parent_command_vars["target_folder"] = default_target if default_target_name: parent_command_vars["target_folder_name"] = default_target_name if default_include: parent_command_vars["includes"] = default_include if default_exclude: parent_command_vars["includes"] = default_include if default_non_recursive is not None: parent_command_vars["non_recursive"] = default_non_recursive freckle_details = [] if not args[0]: # fill missing keys with default values if "target_folder" not in parent_command_vars.keys(): parent_command_vars["target_folder"] = DEFAULT_FRECKLE_TARGET_MARKER if "target_folder_name" not in parent_command_vars.keys(): parent_command_vars["target_folder_name"] = None if "include" not in parent_command_vars.keys(): parent_command_vars["include"] = [] if "exclude" not in parent_command_vars.keys(): parent_command_vars["exclude"] = [] if "non_recursive" not in parent_command_vars.keys(): parent_command_vars["non_recursive"] = False prio = 1000 freckle_repos = [] # TODO: pre-fill with adapter-defaults? for freckle in default_freckle_urls: repo = FreckleRepo(freckle, target_folder=parent_command_vars["target_folder"], target_name=parent_command_vars["target_folder_name"], include=parent_command_vars["include"], exclude=parent_command_vars["exclude"], non_recursive=parent_command_vars["non_recursive"], priority=prio, default_vars={}, overlay_vars={"freckle": default_extra_vars}) prio = prio + 100 freckle_repos.append(repo) details = FreckleDetails(freckle_repos, profiles_to_run=None) freckle_details.append(details) else: multi_freckle_repos = OrderedDict() det_prio = 10000 for p in args[0]: pn = p["name"] # if pn in profiles.keys(): # raise Exception("Profile '{}' specified twice. I don't think that makes sense. Exiting...".format(pn)) metadata = {} metadata = {} metadata["metadata"] = p["adapter_metadata"] metadata["details"] = p["adapter_details"] pvars_adapter_defaults = p["default_vars"] pvars_extra_vars = p["extra_vars"] pvars_user_input = p["user_input"] pvars_profile_extra_vars = pvars_user_input.pop("profile_extra_vars", ()) freckle_default_vars = OrderedDict() for ev in pvars_extra_vars: frkl.dict_merge(freckle_default_vars, ev, copy_dct=False) pvars = OrderedDict() for ev in pvars_profile_extra_vars: frkl.dict_merge(pvars, ev, copy_dct=False) frkl.dict_merge(pvars, pvars_user_input, copy_dct=False) freckles = list(pvars.pop("freckle", [])) include = list(set(pvars.pop("include", []))) exclude = list(set(pvars.pop("exclude", []))) target_folder = pvars.pop("target_folder", None) target_name = pvars.pop("target_name", None) # ask_become_pass = pvars.pop("ask_become_pass", None) # if ask_become_pass is None: # ask_become_pass = default_ask_become_pass non_recursive = pvars.pop("non_recursive", False) if non_recursive is None: non_recursive = default_non_recursive log.debug("Merged vars for profile: freckle".format(pn)) log.debug(readable_json(freckle_default_vars, indent=2)) log.debug("Merged vars for profile: {}".format(pn)) log.debug(readable_json(pvars, indent=2)) all_freckles_for_this_profile = freckles + default_freckle_urls if len(all_freckles_for_this_profile) > 1 and target_name is not None: raise Exception("Can't use 'target_name' if more than one folders are specified") prio = 1000 freckle_repos = [] for freckle in all_freckles_for_this_profile: repo = FreckleRepo(freckle, target_folder=target_folder, target_name=target_name, include=include, exclude=exclude, non_recursive=non_recursive, priority=prio, default_vars={pn: pvars_adapter_defaults}, overlay_vars={pn: pvars, "freckle": freckle_default_vars}) prio = prio + 100 freckle_repos.append(repo) details = FreckleDetails(freckle_repos, profiles_to_run=pn, detail_priority=det_prio) freckle_details.append(details) det_prio = det_prio + 1000 if default_password is None: default_password = "******" if default_password == "ask": password = click.prompt("Please enter sudo password for this run", hide_input=True) click.echo() default_password = False # TODO: check password valid elif default_password == "ansible": default_password = True password = None elif default_password == "no": default_password = False password = None else: raise click.ClickException("Can't process password: {}".format(default_password)) try: f = Freckelize(freckle_details, ask_become_pass=default_password, password=password) f.execute(hosts=hosts, no_run=no_run, output_format=default_output_format) except (Exception) as e: raise click.ClickException(str(e)) sys.exit(0)
def create_and_run_nsbl_runner(task_config, task_metadata={}, output_format="default", ask_become_pass=False, password=None, pre_run_callback=None, no_run=False, additional_roles=[], config=None, run_box_basics=False, additional_repo_paths=[], hosts_list=["localhost"]): if run_box_basics: result = execute_run_box_basics(output_format, ask_become_pass, password) if result["return_code"] > 0: return result if not config: config = DEFAULT_FRECKLES_CONFIG if additional_repo_paths: if config.use_freckle_as_repo: config.add_repo(additional_repo_paths) config_trusted_repos = config.trusted_repos local_role_repos = [] for repo_name in config_trusted_repos: repo = DEFAULT_ROLE_REPOS.get(repo_name) if not repo: if not os.path.exists(repo_name): repo_details = nsbl_tasks.expand_string_to_git_details(repo_name, DEFAULT_ABBREVIATIONS) repo_url = repo_details["url"] repo_branch = repo_details.get('branch', None) relative_repo_path = nsbl_tasks.calculate_local_repo_path(repo_url, repo_branch) repo_path = os.path.join(DEFAULT_LOCAL_REPO_PATH_BASE, relative_repo_path) else: local_role_repos.append(repo_name) else: repo_path = repo["path"] local_role_repos.append(repo_path) role_repos = nsbl_defaults.calculate_role_repos(local_role_repos) task_descs = config.task_descs global_vars = {} for tc in task_config: v = tc.get("vars", {}) if v: dict_merge(global_vars, v, copy_dct=False) nsbl_obj = nsbl.Nsbl.create(task_config, role_repos, task_descs, wrap_into_hosts=hosts_list, pre_chain=[], additional_roles=additional_roles) runner = nsbl.NsblRunner(nsbl_obj) run_target = os.path.expanduser(DEFAULT_RUN_LOCATION) ansible_verbose = "" stdout_callback = "nsbl_internal" ignore_task_strings = [] display_sub_tasks = True display_skipped_tasks = False if output_format == "verbose": stdout_callback = "default" ansible_verbose = "-vvvv" elif output_format == "ansible": stdout_callback = "default" elif output_format == "skippy": stdout_callback = "skippy" elif output_format == "default_full": stdout_callback = "nsbl_internal" display_skipped_tasks = True elif output_format == "default": ignore_task_strings = DEFAULT_IGNORE_STRINGS stdout_callback = "nsbl_internal" elif output_format == "yaml": stdout_callback = "yaml" else: raise Exception("Invalid output format: {}".format(output_format)) force = True extra_paths = "$HOME/.local/share/inaugurate/virtualenvs/freckles/bin:$HOME/.local/share/inaugurate/conda/envs/freckles/bin" return runner.run(run_target, global_vars=global_vars, force=force, ansible_verbose=ansible_verbose, ask_become_pass=ask_become_pass, password=password, extra_plugins=EXTRA_FRECKLES_PLUGINS, callback=stdout_callback, add_timestamp_to_env=True, add_symlink_to_env=DEFAULT_RUN_SYMLINK_LOCATION, no_run=no_run, display_sub_tasks=display_sub_tasks, display_skipped_tasks=display_skipped_tasks, display_ignore_tasks=ignore_task_strings, pre_run_callback=pre_run_callback, extra_paths=extra_paths)
def parse_tasks_dictlet(content, current_vars, tasks_keyword=FX_TASKS_KEY_NAME, vars_keyword=None, delimiter_profile=JINJA_DELIMITER_PROFILES["luci"]): """Process a frecklecutable line-by-line. The main purpose of this is to extract a task list and vars, as well as potential other metadata ('defaults', '__freckle__'). The parsing is a bit convoluted, as the metadata as well as 'vars' can be in a commented section, so 'valid' ansible tasks lists can be used here. The 'vars' and 'tasks' keys need to be at the lasts ones (in this order, preferrably). The 'vars' key can either be in a commented section, or not. If using comments, all the keys in commented section need to have the same comment prefix. """ log.debug("Processing: {}".format(content)) # now, I know this isn't really the most # optimal way of doing this, # but I don't really care that much about execution speed yet, # plus I really want to be able to use variables used in previous # lines of the content last_whitespaces = 0 current_lines = "" temp_vars = copy.deepcopy(current_vars) meta_started = False tasks_started = False tasks_finished = False vars_started = False vars_finished = False vars_meta_started = False vars_meta_finished = False tasks_string = "" vars_string = "" for line in content: # print("LINE: "+line) if not tasks_started and not vars_started: if line.startswith("{}:".format(tasks_keyword)): tasks_started = True continue elif vars_keyword and line.startswith("{}:".format(vars_keyword)): vars_started = True continue if not meta_started: if "defaults:" in line: ignore_prefix = line.partition("defaults:")[0] elif "__freckles__:" in line: ignore_prefix = line.partition("__freckles__:")[0] elif "doc:" in line: ignore_prefix = line.partition("doc:")[0] elif "args:" in line: ignore_prefix = line.partition("args:")[0] elif vars_keyword and "{}:".format(vars_keyword) in line: ignore_prefix = line.partition("vars:")[0] vars_meta_started = True meta_started = True continue else: continue meta_started = True else: if ignore_prefix and not line.startswith( ignore_prefix) and not (ignore_prefix.strip() and line.startswith( ignore_prefix.strip())): meta_started = False continue line_new = line[len(ignore_prefix):] if vars_keyword and line_new.startswith( "{}:".format(vars_keyword)): vars_meta_started = True continue if vars_meta_started: vars_string = "{}\n{}".format(vars_string, line_new) continue whitespaces = len(line_new) - len(line_new.lstrip(' ')) current_lines = "{}{}\n".format(current_lines, line_new) if whitespaces <= last_whitespaces: if delimiter_profile: temp = replace_string(current_lines, temp_vars, **delimiter_profile) else: temp = current_lines if not temp.strip(): continue temp_dict = ordered_load(temp) temp_vars = frkl.dict_merge(temp_vars, temp_dict, copy_dct=False) last_whitespaces = whitespaces else: if vars_keyword and line.startswith("{}:".format(vars_keyword)): if tasks_started: tasks_finished = True tasks_started = False if vars_finished: raise Exception( "Can't have two segments starting with '{}' in frecklecutable." .format(self.vars_keyword)) vars_started = True elif line.startswith("{}:".format(tasks_keyword)): if vars_started: vars_finished = True vars_started = False if tasks_finished: raise Exception( "Can't have two segments starting with '{}' in frecklecutable" .format(self.tasks_keyword)) tasks_started = True else: if vars_started: vars_string = "{}\n{}".format(vars_string, line) elif tasks_started: tasks_string = "{}\n{}".format(tasks_string, line) else: raise Exception( "Internal error in frecklecutable reader. Please report an issue." ) if current_lines: if delimiter_profile: temp = replace_string( current_lines, temp_vars, additional_jinja_extensions=freckles_jinja_extensions, **delimiter_profile) else: temp = current_lines if temp.strip(): temp_dict = ordered_load(temp) temp_vars = frkl.dict_merge(temp_vars, temp_dict, copy_dct=False) frkl.dict_merge(current_vars, temp_vars, copy_dct=False) current_vars[tasks_keyword] = tasks_string current_vars[vars_keyword] = vars_string log.debug("Vars after processing: {}".format(current_vars)) return current_vars
def add_default_vars(self, vars_dict): frkl.dict_merge(self.default_vars, vars_dict, copy_dct=False)
def add_overlay_vars(self, vars_dict): frkl.dict_merge(self.overlay_vars, vars_dict, copy_dct=False)
def render(self, env_dir, extra_plugins=None, extract_vars=True, force=False, ask_become_pass="******", ansible_args="", callback='default', force_update_roles=False, add_timestamp_to_env=False, add_symlink_to_env=False): """Creates the ansible environment in the folder provided. Args: env_dir (str): the folder where the environment should be created extra_plugins (str): a path to a repository of extra ansible plugins, if necessary extract_vars (bool): whether to extract a hostvars and groupvars directory for the inventory (True), or render a dynamic inventory script for the environment (default, True) -- Not supported at the moment force (bool): overwrite environment if already present at the specified location, use with caution because this might delete an important folder if you get the 'target' dir wrong ask_become_pass (str): whether to include the '--ask-become-pass' arg to the ansible-playbook call, options: 'auto', 'true', 'false' ansible_verbose (str): parameters to give to ansible-playbook (like: "-vvv") callback (str): name of the callback to use, default: nsbl_internal force_update_roles (bool): whether to overwrite external roles that were already downloaded add_timestamp_to_env (bool): whether to add a timestamp to the env_dir -- useful for when this is called from other programs (e.g. freckles) add_symlink_to_env (bool): whether to add a symlink to the current env from a fixed location (useful to archive all runs/logs) """ if isinstance(ask_become_pass, bool): ask_become_pass = str(ask_become_pass) if not isinstance(ask_become_pass, string_types) or ask_become_pass.lower() not in [ "true", "false", "auto" ]: raise NsblException( "Can't parse 'ask_become_pass' var: '{}'".format( ask_become_pass)) if ask_become_pass == "auto": ask_become_pass = self.use_become else: ask_become_pass = ask_become_pass.lower() in (['true', 'yes']) env_dir = os.path.expanduser(env_dir) if add_timestamp_to_env: start_date = datetime.now() date_string = start_date.strftime('%y%m%d_%H_%M_%S') dirname, basename = os.path.split(env_dir) env_dir = os.path.join(dirname, "{}_{}".format(basename, date_string)) result = {} result['env_dir'] = env_dir if os.path.exists(env_dir) and force: shutil.rmtree(env_dir) inventory_dir = os.path.join(env_dir, "inventory") result["inventory_dir"] = inventory_dir if extract_vars: inv_target = "../inventory/hosts" else: inv_target = "../inventory/inventory" result["extract_vars"] = extract_vars playbook_dir = os.path.join(env_dir, "plays") result["playbook_dir"] = playbook_dir roles_base_dir = os.path.join(env_dir, "roles") result["roles_base_dir"] = playbook_dir # ask_sudo = "--ask-become-pass" if ask_become_pass: if can_passwordless_sudo(): ask_sudo = "" else: ask_sudo = "--ask-become-pass" else: ask_sudo = "" all_plays_name = "all_plays.yml" result["default_playbook_name"] = all_plays_name ansible_playbook_args = ansible_args result["ansible_playbook_cli_args"] = ansible_playbook_args result["run_playbooks_script"] = os.path.join(env_dir, "run_all_plays.sh") try: import ara ara_base = os.path.dirname(ara.__file__) except: ara_base = None pass if ara_base: callback_plugins_list = "callback_plugins:{}/plugins/callbacks".format( ara_base) action_plugins_list = "action_plugins:{}/plugins/actions".format( ara_base) library_plugins_list = "library:{}/plugins/modules".format( ara_base) else: callback_plugins_list = "callback_plugins" action_plugins_list = "action_plugins" library_plugins_list = "library" cookiecutter_details = { "inventory": inv_target, "env_dir": env_dir, "playbook_dir": playbook_dir, "ansible_playbook_args": ansible_playbook_args, "library_path": library_plugins_list, "action_plugins_path": action_plugins_list, "extra_script_commands": "", "ask_sudo": ask_sudo, "playbook": all_plays_name, "callback_plugins": callback_plugins_list, "callback_plugin_name": callback, "callback_whitelist": "default_to_file" } log.debug("Creating build environment from template...") log.debug( "Using cookiecutter details: {}".format(cookiecutter_details)) template_path = os.path.join(os.path.dirname(__file__), "external", "cookiecutter-ansible-environment") cookiecutter(template_path, extra_context=cookiecutter_details, no_input=True) if add_symlink_to_env: link_path = os.path.expanduser(add_symlink_to_env) if os.path.exists(link_path) and force: os.unlink(link_path) link_parent = os.path.abspath(os.path.join(link_path, os.pardir)) try: os.makedirs(link_parent) except: pass os.symlink(env_dir, link_path) # write inventory if extract_vars: self.inventory.extract_vars(inventory_dir) self.inventory.write_inventory_file_or_script( inventory_dir, extract_vars=extract_vars) # write roles all_playbooks = [] ext_roles = False roles_to_copy = {} task_details = [] for play, tasks in self.plays.items(): task_details.append(str(tasks)) playbook = tasks.render_playbook(playbook_dir) all_playbooks.append(playbook) tasks.render_roles(roles_base_dir) if tasks.roles_to_copy: dict_merge(roles_to_copy, tasks.roles_to_copy, copy_dct=False) if tasks.ext_roles: ext_roles = True result["task_details"] = task_details jinja_env = Environment(loader=PackageLoader('nsbl', 'templates')) template = jinja_env.get_template('play.yml') output_text = template.render(playbooks=all_playbooks) all_plays_file = os.path.join(env_dir, "plays", all_plays_name) result["all_plays_file"] = all_plays_file with open(all_plays_file, "w") as text_file: text_file.write(output_text) # copy extra_plugins library_path = os.path.join(os.path.dirname(__file__), "external", "extra_plugins", "library") action_plugins_path = os.path.join(os.path.dirname(__file__), "external", "extra_plugins", "action_plugins") callback_plugins_path = os.path.join(os.path.dirname(__file__), "external", "extra_plugins", "callback_plugins") result["library_path"] = library_path result["callback_plugins_path"] = callback_plugins_path result["action_plugins_path"] = action_plugins_path target_dir = playbook_dir if extra_plugins: dirs = [ o for o in os.listdir(extra_plugins) if os.path.isdir(os.path.join(extra_plugins, o)) ] for d in dirs: shutil.copytree(os.path.join(extra_plugins, d), os.path.join(target_dir, d)) if ext_roles: # download external roles click.echo("\nDownloading external roles...") role_requirement_file = os.path.join(env_dir, "roles", "roles_requirements.yml") if not os.path.exists(ANSIBLE_ROLE_CACHE_DIR): os.makedirs(ANSIBLE_ROLE_CACHE_DIR) command = [ "ansible-galaxy", "install", "-r", role_requirement_file, "-p", ANSIBLE_ROLE_CACHE_DIR ] if force_update_roles: command.append("--force") log.debug("Downloading and installing external roles...") my_env = os.environ.copy() my_env["PATH"] = "{}:{}:{}".format( os.path.expanduser("~/.local/bin"), os.path.expanduser("~/.local/inaugurate/bin"), my_env["PATH"]) res = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, env=my_env) for line in iter(res.stdout.readline, ""): if "already installed" not in line and "--force to change" not in line: # log.debug("Installing role: {}".format(line.encode('utf8'))) click.echo(" {}".format(line.encode('utf8')), nl=False) if roles_to_copy.get("internal", {}): for src, target in roles_to_copy["internal"].items(): log.debug("Coping internal role: {} -> {}".format(src, target)) shutil.copytree(src, target) if roles_to_copy.get("external", {}): for src, target in roles_to_copy["external"].items(): log.debug("Coping external role: {} -> {}".format(src, target)) shutil.copytree(src, target) return result
def create_freckle_descs(repos, config=None): """Augments freckle urls provided by the user via cli (if necessary). """ for temp_url, metadata in repos.items(): target = metadata["target_folder"] source_delete = False if target != DEFAULT_FRECKLE_TARGET_MARKER and not target.startswith( "~") and not target.startswith("/"): raise Exception( "Relative path not supported for target option, please use an absolute one" ) if target == DEFAULT_FRECKLE_TARGET_MARKER or target.startswith( "~") or target.startswith("/home"): target_become = False else: target_become = True if temp_url.startswith(BLUEPRINT_URL_PREFIX): blueprint_name = ":".join(temp_url.split(":")[1:]) blueprints = get_available_blueprints(config) match = blueprints.get(blueprint_name, False) if not match: raise Exception( "No blueprint with name '{}' available.".format( blueprint_name)) if target == DEFAULT_FRECKLE_TARGET_MARKER: raise Exception( "No target directory specified when using blueprint.") cookiecutter_file = os.path.join(match, "cookiecutter.json") if os.path.exists(cookiecutter_file): click.secho( "\nFound interactive blueprint, please enter approriate values below:\n", bold=True) temp_path = tempfile.mkdtemp(prefix='frkl.') cookiecutter(match, output_dir=temp_path) subdirs = [ os.path.join(temp_path, f) for f in listdir(temp_path) if isdir(join(temp_path, f)) ] if len(subdirs) != 1: raise Exception( "More than one directories created by interactive template '{}'. Can't deal with that." .format(match)) url = subdirs[0] source_delete = True else: url = match else: url = temp_url if os.path.exists(os.path.expanduser(url)): # assuming archive if os.path.isfile(os.path.expanduser(url)): # TODO: check whether host is local repo_desc = {"type": "local_archive"} repo_desc["remote_url"] = os.path.abspath( os.path.expanduser(url)) repo_desc["checkout_become"] = target_become #repo_desc["local_name"] = os.path.basename(url).split('.')[0] repo_desc["source_delete"] = False if target == DEFAULT_FRECKLE_TARGET_MARKER: raise Exception( "No target directory specified when using archive file." ) repo_desc["local_parent"] = target repo_desc["checkout_skip"] = False else: # TODO: check whether host is local repo_desc = {"type": "local"} repo_desc["remote_url"] = os.path.abspath( os.path.expanduser(url)) repo_desc["checkout_become"] = target_become repo_desc["local_name"] = os.path.basename( repo_desc["remote_url"]) repo_desc["source_delete"] = source_delete if target == DEFAULT_FRECKLE_TARGET_MARKER: repo_desc["checkout_skip"] = True repo_desc["local_parent"] = os.path.dirname( repo_desc["remote_url"]) else: repo_desc["local_parent"] = target repo_desc["checkout_skip"] = False elif url.endswith(".git"): repo_desc = {"type": "git"} repo_desc["remote_url"] = url repo_desc["checkout_become"] = target_become repo_desc["local_name"] = os.path.basename( repo_desc["remote_url"])[:-4] repo_desc["checkout_skip"] = False if target == DEFAULT_FRECKLE_TARGET_MARKER: repo_desc["local_parent"] = "~/freckles" else: repo_desc["local_parent"] = target elif url.startswith("http://") or url.startswith("https://"): # TODO: check whether host is local repo_desc = {"type": "remote_archive"} repo_desc["remote_url"] = url repo_desc["checkout_become"] = target_become #repo_desc["local_name"] = os.path.basename(url).split('.')[0] repo_desc["source_delete"] = False if target == DEFAULT_FRECKLE_TARGET_MARKER: raise Exception( "No target directory specified when using archive file.") repo_desc["local_parent"] = target repo_desc["checkout_skip"] = False else: raise Exception( "freckle url format unknown, don't know how to handle that: {}" .format(url)) repo_desc["blueprint"] = temp_url.startswith(BLUEPRINT_URL_PREFIX) frkl.dict_merge(metadata, repo_desc, copy_dct=False)
def __init__(self, name, tasks, vars, tasks_format=None, external_task_list_map={}, additional_roles=[]): self.name = name self.tasks = tasks if isinstance(tasks, string_types): self.tasks = yaml.safe_load(tasks) self.tasks_string = tasks elif isinstance(tasks, (list, tuple)): self.tasks = tasks self.tasks_string = yaml.safe_dump(tasks, default_flow_style=False) else: raise Exception("Invalid type for tasks list: {}".format( type(tasks))) self.vars = vars self.metadata = {} # not used at the moment if tasks_format == None: log.debug("Trying to guess task-list format...") tasks_format = get_task_list_format(self.tasks) if tasks_format == None: log.info( "Could not determine task list format for sure, falling back to '{}'." .format(DEFAULT_FRECKLECUTALBE_TASK_LIST_FORMAT)) tasks_format = DEFAULT_FRECKLECUTALBE_TASK_LIST_FORMAT self.tasks_format = tasks_format if self.tasks_format not in ["freckles", "ansible"]: raise Exception("Invalid task-list format: {}".format( self.tasks_format)) self.external_task_list_map = external_task_list_map # getting ansible roles, this is not necessary for the 'freckles' configuration format self.additional_roles = additional_roles self.task_list_aliases = {} for name, details in self.external_task_list_map.items(): self.task_list_aliases[name] = details["play_target"] # generating rendered tasks depending on task-list format if self.tasks_format == "ansible": relative_target_file = os.path.join( "{{ playbook_dir }}", "..", "task_lists", "frecklecutable_default_tasks.yml") self.final_tasks = [{ "meta": { "name": "include_tasks", "task-desc": "[including tasks]", "var-keys": ["free_form"], }, "vars": { "free_form": relative_target_file } }] else: self.final_tasks = self.tasks self.all_vars = frkl.dict_merge(vars, self.task_list_aliases, copy_dct=True) self.task_config = [{"tasks": self.final_tasks, "vars": self.all_vars}]
def freckles_process(self, command_name, default_vars, extra_vars, user_input, metadata, dictlet_details, config, parent_params, command_var_spec): all_vars = OrderedDict() frkl.dict_merge(all_vars, default_vars, copy_dct=False) for ev in extra_vars: frkl.dict_merge(all_vars, ev, copy_dct=False) frkl.dict_merge(all_vars, user_input, copy_dct=False) hosts = parent_params.get("hosts", ["localhost"]) output_format = parent_params.get("output", "default") password_type = parent_params.get("password", None) no_run = parent_params.get("no_run", False) tasks_string = metadata.get(FX_TASKS_KEY_NAME, "") vars_string = metadata.get(FX_VARS_KEY_NAME, "") replaced_vars = replace_string( vars_string, all_vars, additional_jinja_extensions=freckles_jinja_extensions, **JINJA_DELIMITER_PROFILES["luci"]) try: vars_dictlet = yaml.safe_load(replaced_vars) except (Exception) as e: raise Exception("Can't parse vars: {}".format(e)) if vars_dictlet: temp_new_all_vars = frkl.dict_merge(all_vars, vars_dictlet, copy_dct=True) else: temp_new_all_vars = all_vars replaced_tasks = replace_string( tasks_string, temp_new_all_vars, additional_jinja_extensions=freckles_jinja_extensions, **JINJA_DELIMITER_PROFILES["luci"]) try: tasks_list_temp = ordered_load(replaced_tasks) except (Exception) as e: raise click.ClickException( "Could not parse frecklecutable '{}': {}".format( command_name, e)) extra_task_lists_map = process_extra_task_lists( metadata, dictlet_details["path"]) # check for hardcoded task_list_format: task_list_format = metadata.get("__freckles__", {}).get("task_list_format", None) additional_roles = metadata.get("__freckles__", {}).get("roles", []) result_vars = {} for name, details in command_var_spec.items(): if name in temp_new_all_vars and details.get("is_var", False) == True: result_vars[name] = temp_new_all_vars[name] f = Frecklecutable(command_name, replaced_tasks, result_vars, tasks_format=task_list_format, external_task_list_map=extra_task_lists_map, additional_roles=additional_roles) # placeholder, for maybe later task_metadata = {} if password_type is None: password_type = "no" if password_type == "ask": password = click.prompt("Please enter sudo password for this run", hide_input=True) click.echo() password_type = False # TODO: check password valid elif password_type == "ansible": password_type = True password = None elif password_type == "no": password_type = False password = None else: raise Exception("Can't process password: {}".format(password_type)) run = Frecklecute(f, config=self.config, ask_become_pass=password_type, password=password) run.execute(hosts=hosts, no_run=no_run, output_format=output_format)