def _load_and_merge_config_files(config_files=[]): """ Walk through our found config files and load them into a set :param config_files: List of config files :return: set of config files """ merged_config = {} for config_file in config_files: config_filename = os.path.expanduser(config_file.get('path')) if os.path.exists(config_filename): try: loaded_config = _load_config_file(config_filename) print(green("Loaded: {}".format(config_filename))) if config_file.get('preferred'): print(red("Deprecated location for {} - Please use {}".format(config_filename, config_file.get('preferred')))) merged_config = dict_deepmerge(loaded_config, merged_config) except Exception as e: if 'preferred' not in config_file: print(yellow("Warning - error loading config: {}".format(config_filename))) print(yellow(e)) else: # let's only print preferred locations that we skipped if 'preferred' not in config_file: print("Skipped: {}".format(config_filename)) return merged_config
def _load_and_merge_config_files(config_files=[]): """ Walk through our found config files and load them into a set :param config_files: List of config files :return: set of config files """ merged_config = {} for config_file in config_files: config_filename = os.path.expanduser(config_file.get('path')) if os.path.exists(config_filename): try: loaded_config = _load_config_file(config_filename) print(green("Loaded: {}".format(config_filename))) if config_file.get('preferred'): print( red("Deprecated location for {} - Please use {}". format(config_filename, config_file.get('preferred')))) merged_config = dict_deepmerge(loaded_config, merged_config) except Exception as e: if 'preferred' not in config_file: print( yellow("Warning - error loading config: {}".format( config_filename))) print(yellow(e)) else: # let's only print preferred locations that we skipped if 'preferred' not in config_file: print("Skipped: {}".format(config_filename)) return merged_config
def parse_salt_output(output_salt, parsed_summary): """ Parse our salt output from a yamly file and return a summary :param output_salt: string, yaml contents :param parsed_summary: list :return: int, the number of calls that failed """ failed = 0 changed = 0 worked = 0 salt_yml = yaml_ordered_load(output_salt) for salted_host in salt_yml: host_fail = 0 host_work = 0 host_change = 0 parsed_summary.append(blue("\n{}:".format(salted_host), bold=True)) for salt_event in salt_yml[salted_host]: event = salt_yml[salted_host][salt_event] for salt_event_type in event: if salt_event_type == "result": if event[salt_event_type]: worked += 1 host_work += 1 else: failed += 1 host_fail += 1 parsed_summary.append( red("\tFailure: {}".format(salt_event), bold=True)) parsed_summary.append( red("\t Reason: {}".format(event['comment']), bold=False)) parsed_summary.append( red("\t Debug: {}".format(event), bold=False)) elif salt_event_type == "changes" and len( event[salt_event_type]): changed += 1 host_change += 1 parsed_summary.append( white("\tChange: {name} - Comment: {comment}".format( name=event['name'], comment=event['comment']), bold=False)) parsed_summary.append( yellow("\tSuccess: {}".format(host_work), bold=False)) parsed_summary.append( white("\tChanged: {}".format(host_change), bold=False)) parsed_summary.append(red("\tFailed: {}".format(host_fail), bold=False)) parsed_summary.append(blue("\nSummary:", bold=True)) parsed_summary.append(yellow("\tSuccess: {}".format(worked), bold=True)) parsed_summary.append(white("\tChanged: {}".format(changed), bold=True)) parsed_summary.append(red("\tFailed: {}".format(failed), bold=True)) return failed
def _git_commit(self, changes=[], message=''): index = self.repository.index print(yellow("Staging change-set")) for change in changes: self.repository.git.add(change) print(yellow("Staged: {}".format(change))) print(yellow("Committing files with message: {}".format(message))) index.commit(message, author=self.author, committer=self.author)
def parse_salt_output(output_salt, parsed_summary): """ Parse our salt output from a yamly file and return a summary :param output_salt: string, yaml contents :param parsed_summary: list :return: int, the number of calls that failed """ failed = 0 changed = 0 worked = 0 salt_yml = yaml_ordered_load(output_salt) for salted_host in salt_yml: host_fail = 0 host_work = 0 host_change = 0 parsed_summary.append(blue("\n{}:".format(salted_host), bold=True)) for salt_event in salt_yml[salted_host]: event = salt_yml[salted_host][salt_event] for salt_event_type in event: if salt_event_type == "result": if event[salt_event_type]: worked += 1 host_work += 1 else: failed += 1 host_fail += 1 parsed_summary.append(red("\tFailure: {}".format(salt_event), bold=True)) parsed_summary.append(red("\t Reason: {}".format(event['comment']), bold=False)) parsed_summary.append(red("\t Debug: {}".format(event), bold=False)) elif salt_event_type == "changes" and len(event[salt_event_type]): changed += 1 host_change += 1 parsed_summary.append( white("\tChange: {name} - Comment: {comment}".format( name=event['name'], comment=event['comment']), bold=False)) parsed_summary.append(yellow("\tSuccess: {}".format(host_work), bold=False)) parsed_summary.append(white("\tChanged: {}".format(host_change), bold=False)) parsed_summary.append(red("\tFailed: {}".format(host_fail), bold=False)) parsed_summary.append(blue("\nSummary:", bold=True)) parsed_summary.append(yellow("\tSuccess: {}".format(worked), bold=True)) parsed_summary.append(white("\tChanged: {}".format(changed), bold=True)) parsed_summary.append(red("\tFailed: {}".format(failed), bold=True)) return failed
def refresh_pillars(selector="'*'", prefix='', salt_environment=None): """ Wrapper for clear_cache and reload_pillar :param selector: String selector :param prefix: String used for compound or grain matches :param salt_environment: string representing our target environment :return: """ print yellow("Clearing pillar cache") clear_cache(selector=selector, prefix=prefix, salt_environment=salt_environment) print yellow("Refreshing pillar data") reload_pillar(selector=selector, prefix=prefix, salt_environment=salt_environment)
def filter(self, **kwargs): """ return: list of objects matching filter args typically provide should support filter 'name'='foo' """ instances = [] if 'name' in kwargs: name = kwargs['name'] selected = False for reservation in self.connection.get_all_instances(): instance = reservation.instances[0] if instance.state not in ['terminated', 'shutting-down']: if "Name" in instance.tags and instance.tags[ "Name"] == name: instances.append(instance) selected = True print( green("Selected aws instance: {}".format( instance.id))) if not selected: print(yellow("Warning: {} not found!".format(name), bold=True)) else: raise NotImplementedError() return instances
def highstate_complete(): """ Poll jobs active waiting for it to become empty after an unattended_highstate was started """ timeout = 15 result = StringIO() salt_run(method='jobs.active', stdout=result) while len(result.getvalue().strip()): result.truncate(0) print yellow("Highstate is still running.\nPolling again in {} seconds.\n".format(timeout)) time.sleep(timeout) salt_run(method='jobs.active', stdout=result) print green("Highstate complete.\n") result.close()
def _stash_changes(self): self.stashed_changes = False result = self._git_status() if 'nothing to commit, working directory clean' not in result: print(yellow("Stashing changes")) self.repository.git.stash() self.stashed_changes = True
def highstate_complete(): """ Poll jobs active waiting for it to become empty after an unattended_highstate was started """ timeout = 15 result = StringIO() salt_run(method='jobs.active', stdout=result) while len(result.getvalue().strip()): result.truncate(0) print yellow( "Highstate is still running.\nPolling again in {} seconds.\n". format(timeout)) time.sleep(timeout) salt_run(method='jobs.active', stdout=result) print green("Highstate complete.\n") result.close()
def refresh_pillars(selector="'*'", prefix='', salt_environment=None): """ Wrapper for clear_cache and reload_pillar :param selector: String selector :param prefix: String used for compound or grain matches :param salt_environment: string representing our target environment :return: """ print yellow("Clearing pillar cache") clear_cache( selector=selector, prefix=prefix, salt_environment=salt_environment ) print yellow("Refreshing pillar data") reload_pillar( selector=selector, prefix=prefix, salt_environment=salt_environment )
def get_rendered_pillar_location( pillar_dir=None, projects_location=None, parse_top_sls=True, target=None ): """ Returns path to rendered pillar. Use to render pillars written in jinja locally not to upload unwanted data to network. i.e. you can use constructs like: {% include 'opg-lpa-dev/pillar/services.sls' %} If you want salt to later render pillars with grain context use constructs like: {% raw %} {{grains.get('roles')}} {% endraw %} {{" {{grains.get('roles')}} "}} To allow for server side templating of top.sls, you will need set: `parse_top_sls=False` In case there is no top.sls in pillar root than it returns: None """ if 'use_project_dir' not in 'env': env.use_project_dir = False if projects_location is None: projects_location = _get_projects_location() pillars = __load_pillar_dirs(pillar_dir, projects_location, target) jinja_env = _set_template_env(pillars, projects_location) dest_location = tempfile.mkdtemp() if parse_top_sls: files_to_render = _pillar_from_top_sls(dest_location, jinja_env) elif len(pillars): files_to_render, output_dir = _pillar_from_dirs(pillars) else: print(yellow("No template files where found to render")) files_to_render = [] if __render_templates(files_to_render, dest_location, jinja_env) is False: print(red("Aborting due to pillar failing to render")) exit(-1) return dest_location
def get_rendered_pillar_location(pillar_dir=None, projects_location=None, parse_top_sls=True, target=None): """ Returns path to rendered pillar. Use to render pillars written in jinja locally not to upload unwanted data to network. i.e. you can use constructs like: {% include 'opg-lpa-dev/pillar/services.sls' %} If you want salt to later render pillars with grain context use constructs like: {% raw %} {{grains.get('roles')}} {% endraw %} {{" {{grains.get('roles')}} "}} To allow for server side templating of top.sls, you will need set: `parse_top_sls=False` In case there is no top.sls in pillar root than it returns: None """ if 'use_project_dir' not in 'env': env.use_project_dir = False if projects_location is None: projects_location = _get_projects_location() pillars = __load_pillar_dirs(pillar_dir, projects_location, target) jinja_env = _set_template_env(pillars, projects_location) dest_location = tempfile.mkdtemp() if parse_top_sls: files_to_render = _pillar_from_top_sls(dest_location, jinja_env) elif len(pillars): files_to_render, output_dir = _pillar_from_dirs(pillars) else: print(yellow("No template files where found to render")) files_to_render = [] if __render_templates(files_to_render, dest_location, jinja_env) is False: print(red("Aborting due to pillar failing to render")) exit(-1) return dest_location
def commit_change_set(self): files_exist = [] for change in self.change_set: if change in self._git_status(): files_exist.append(change) if len(files_exist): try: self._stash_changes() self._checkout_branch() self._pull_branch() self._pop_changes() self._git_commit(files_exist, self.message) except GitCommandError as e: if not e.stderr.contains('No stash found.'): print(red("Git returned: {}".format(e.stderr))) exit(e.status) else: print(yellow(self.NO_CHANGES_TO_COMMIT)) return self.NO_CHANGES_TO_COMMIT
def filter(self, **kwargs): """ return: list of objects matching filter args typically provide should support filter 'name'='foo' """ instances = [] if 'name' in kwargs: name = kwargs['name'] zone_config = get_provider_zone_config() assert zone_config['driver'] == 'static' for host_spec in zone_config['hosts']: if host_spec['name'] == name: print("selected static instance: {}".format(host_spec['name'])) instances.append(host_spec) break else: print(yellow("Warning: {} not found!".format(name), bold=True)) else: raise NotImplementedError() return instances
def filter(self, **kwargs): """ return: list of objects matching filter args typically provide should support filter 'name'='foo' """ instances = [] if 'name' in kwargs: name = kwargs['name'] selected = False for reservation in self.connection.get_all_instances(): instance = reservation.instances[0] if instance.state not in ['terminated', 'shutting-down']: if "Name" in instance.tags and instance.tags["Name"] == name: instances.append(instance) selected = True print(green("Selected aws instance: {}".format(instance.id))) if not selected: print(yellow("Warning: {} not found!".format(name), bold=True)) else: raise NotImplementedError() return instances
def filter(self, **kwargs): """ return: list of objects matching filter args typically provide should support filter 'name'='foo' """ instances = [] if 'name' in kwargs: name = kwargs['name'] zone_config = get_provider_zone_config() assert zone_config['driver'] == 'static' for host_spec in zone_config['hosts']: if host_spec['name'] == name: print("selected static instance: {}".format( host_spec['name'])) instances.append(host_spec) break else: print(yellow("Warning: {} not found!".format(name), bold=True)) else: raise NotImplementedError() return instances
def build_salt_dirs(): import yaml """ get list of pillar directories for cotton :return: dict with list of dirs or error """ print(yellow("{}".format('Building pillar directory list'))) try: dir_list = [] not_found = [] root_list = [] with open('sources.yml') as f: _dirs = yaml.safe_load(f) pillar_dirs = _dirs['pillar'][ env.provider_zone] + _dirs['pillar']['common'] pillar_roots = _dirs['pillar_roots'][ env.provider_zone] + _dirs['pillar_roots']['common'] for pillar_dir_name in pillar_dirs: if os.path.exists(pillar_dir_name): dir_list.append(pillar_dir_name) else: not_found.append(pillar_dir_name) for pillar_rootdir_name in pillar_roots: if os.path.exists(pillar_rootdir_name): root_list.append(pillar_rootdir_name) else: not_found.append(pillar_rootdir_name) return { 'pillar_dir': dir_list, 'pillar_root': root_list, 'missing': not_found } except (OSError, IOError): return {'error': 'Failed to open sources.yml'}
def _pop_changes(self): if self.stashed_changes: print(yellow("Popping changes")) self.repository.git.stash('pop')
def _checkout_branch(self): print(yellow("Checking out {}".format(self.target_branch))) self.repository.git.checkout(self.target_branch)
def get_rendered_pillar_location(): """ Returns path to rendered pillar. Use to render pillars written in jinja locally not to upload unwanted data to network. i.e. you can use constructs like: {% include 'opg-lpa-dev/pillar/services.sls' %} In case there is no top.sls in pillar root than it returns: None """ from jinja2 import Environment from jinja2 import FileSystemLoader from jinja2.exceptions import TemplateNotFound assert env.project projects_location = _get_projects_location() jinja_env = Environment(loader=FileSystemLoader([ os.path.join(projects_location, env.project, 'pillar'), projects_location ])) # let's get rendered top.sls for configured project try: top_sls = jinja_env.get_template('top.sls').render(env=env) except TemplateNotFound: print(red("Missing top.sls in pillar location. Skipping rendering.")) return None top_content = yaml.load(top_sls) dest_location = tempfile.mkdtemp() with open(os.path.join(dest_location, 'top.sls'), 'w') as f: f.write(top_sls) # get list of files referenced by top.sls files_to_render = [] for k0, v0 in top_content.iteritems(): for k1, v1 in v0.iteritems(): for file_short in v1: # We force this file to be relative in case jinja failed rendering # a variable. This would make the filename start with / and instead of # writing under dest_location it will try to write in / files_to_render.append('./' + file_short.replace('.', '/') + '.sls') # render and save templates for template_file in files_to_render: filename = os.path.abspath(os.path.join(dest_location, template_file)) print( yellow("Pillar template_file: {} --> {}".format( template_file, filename))) if not os.path.isdir(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename)) try: template_rendered = jinja_env.get_template(template_file).render( env=env) except TemplateNotFound: template_rendered = '' print( yellow("Pillar template_file not found: {} --> {}".format( template_file, filename))) with open(os.path.join(dest_location, template_file), 'w') as f: f.write(template_rendered) print(green("Pillar was rendered in: {}".format(dest_location))) return dest_location
def _pull_branch(self): print(yellow("Rebasing against branch")) self.repository.git.pull('--rebase')
def _reset_uncommitted_files(self): print(yellow("Resetting uncommitted files")) self.repository.git.checkout('.')
def _clean_unstaged_files(self): print(yellow("Cleaning up unstaged files")) for untracked_file in self.repository.untracked_files: os.unlink(untracked_file)