def _load_and_merge_config_files(config_files=[]):
"""
Walk through our found config files and load them into a set
:param config_files: List of config files
:return: set of config files
"""
merged_config = {}
for config_file in config_files:
config_filename = os.path.expanduser(config_file.get('path'))
if os.path.exists(config_filename):
try:
loaded_config = _load_config_file(config_filename)
print(green("Loaded: {}".format(config_filename)))
if config_file.get('preferred'):
print(red("Deprecated location for {} - Please use {}".format(config_filename,
config_file.get('preferred'))))
merged_config = dict_deepmerge(loaded_config, merged_config)
except Exception as e:
if 'preferred' not in config_file:
print(yellow("Warning - error loading config: {}".format(config_filename)))
print(yellow(e))
else:
# let's only print preferred locations that we skipped
if 'preferred' not in config_file:
print("Skipped: {}".format(config_filename))
return merged_config
def _load_and_merge_config_files(config_files=[]):
"""
Walk through our found config files and load them into a set
:param config_files: List of config files
:return: set of config files
"""
merged_config = {}
for config_file in config_files:
config_filename = os.path.expanduser(config_file.get('path'))
if os.path.exists(config_filename):
try:
loaded_config = _load_config_file(config_filename)
print(green("Loaded: {}".format(config_filename)))
if config_file.get('preferred'):
print(
red("Deprecated location for {} - Please use {}".
format(config_filename,
config_file.get('preferred'))))
merged_config = dict_deepmerge(loaded_config, merged_config)
except Exception as e:
if 'preferred' not in config_file:
print(
yellow("Warning - error loading config: {}".format(
config_filename)))
print(yellow(e))
else:
# let's only print preferred locations that we skipped
if 'preferred' not in config_file:
print("Skipped: {}".format(config_filename))
return merged_config
def parse_salt_output(output_salt, parsed_summary):
"""
Parse our salt output from a yamly file and return a summary
:param output_salt: string, yaml contents
:param parsed_summary: list
:return: int, the number of calls that failed
"""
failed = 0
changed = 0
worked = 0
salt_yml = yaml_ordered_load(output_salt)
for salted_host in salt_yml:
host_fail = 0
host_work = 0
host_change = 0
parsed_summary.append(blue("\n{}:".format(salted_host), bold=True))
for salt_event in salt_yml[salted_host]:
event = salt_yml[salted_host][salt_event]
for salt_event_type in event:
if salt_event_type == "result":
if event[salt_event_type]:
worked += 1
host_work += 1
else:
failed += 1
host_fail += 1
parsed_summary.append(
red("\tFailure: {}".format(salt_event), bold=True))
parsed_summary.append(
red("\t Reason: {}".format(event['comment']),
bold=False))
parsed_summary.append(
red("\t Debug: {}".format(event), bold=False))
elif salt_event_type == "changes" and len(
event[salt_event_type]):
changed += 1
host_change += 1
parsed_summary.append(
white("\tChange: {name} - Comment: {comment}".format(
name=event['name'], comment=event['comment']),
bold=False))
parsed_summary.append(
yellow("\tSuccess: {}".format(host_work), bold=False))
parsed_summary.append(
white("\tChanged: {}".format(host_change), bold=False))
parsed_summary.append(red("\tFailed: {}".format(host_fail),
bold=False))
parsed_summary.append(blue("\nSummary:", bold=True))
parsed_summary.append(yellow("\tSuccess: {}".format(worked), bold=True))
parsed_summary.append(white("\tChanged: {}".format(changed), bold=True))
parsed_summary.append(red("\tFailed: {}".format(failed), bold=True))
return failed
def _git_commit(self, changes=[], message=''):
index = self.repository.index
print(yellow("Staging change-set"))
for change in changes:
self.repository.git.add(change)
print(yellow("Staged: {}".format(change)))
print(yellow("Committing files with message: {}".format(message)))
index.commit(message, author=self.author, committer=self.author)
def parse_salt_output(output_salt, parsed_summary):
"""
Parse our salt output from a yamly file and return a summary
:param output_salt: string, yaml contents
:param parsed_summary: list
:return: int, the number of calls that failed
"""
failed = 0
changed = 0
worked = 0
salt_yml = yaml_ordered_load(output_salt)
for salted_host in salt_yml:
host_fail = 0
host_work = 0
host_change = 0
parsed_summary.append(blue("\n{}:".format(salted_host), bold=True))
for salt_event in salt_yml[salted_host]:
event = salt_yml[salted_host][salt_event]
for salt_event_type in event:
if salt_event_type == "result":
if event[salt_event_type]:
worked += 1
host_work += 1
else:
failed += 1
host_fail += 1
parsed_summary.append(red("\tFailure: {}".format(salt_event), bold=True))
parsed_summary.append(red("\t Reason: {}".format(event['comment']), bold=False))
parsed_summary.append(red("\t Debug: {}".format(event), bold=False))
elif salt_event_type == "changes" and len(event[salt_event_type]):
changed += 1
host_change += 1
parsed_summary.append(
white("\tChange: {name} - Comment: {comment}".format(
name=event['name'],
comment=event['comment']), bold=False))
parsed_summary.append(yellow("\tSuccess: {}".format(host_work), bold=False))
parsed_summary.append(white("\tChanged: {}".format(host_change), bold=False))
parsed_summary.append(red("\tFailed: {}".format(host_fail), bold=False))
parsed_summary.append(blue("\nSummary:", bold=True))
parsed_summary.append(yellow("\tSuccess: {}".format(worked), bold=True))
parsed_summary.append(white("\tChanged: {}".format(changed), bold=True))
parsed_summary.append(red("\tFailed: {}".format(failed), bold=True))
return failed
def refresh_pillars(selector="'*'", prefix='', salt_environment=None):
"""
Wrapper for clear_cache and reload_pillar
:param selector: String selector
:param prefix: String used for compound or grain matches
:param salt_environment: string representing our target environment
:return:
"""
print yellow("Clearing pillar cache")
clear_cache(selector=selector,
prefix=prefix,
salt_environment=salt_environment)
print yellow("Refreshing pillar data")
reload_pillar(selector=selector,
prefix=prefix,
salt_environment=salt_environment)
def filter(self, **kwargs):
"""
return: list of objects matching filter args
typically provide should support filter 'name'='foo'
"""
instances = []
if 'name' in kwargs:
name = kwargs['name']
selected = False
for reservation in self.connection.get_all_instances():
instance = reservation.instances[0]
if instance.state not in ['terminated', 'shutting-down']:
if "Name" in instance.tags and instance.tags[
"Name"] == name:
instances.append(instance)
selected = True
print(
green("Selected aws instance: {}".format(
instance.id)))
if not selected:
print(yellow("Warning: {} not found!".format(name), bold=True))
else:
raise NotImplementedError()
return instances
def highstate_complete():
"""
Poll jobs active waiting for it to become empty after an unattended_highstate was started
"""
timeout = 15
result = StringIO()
salt_run(method='jobs.active', stdout=result)
while len(result.getvalue().strip()):
result.truncate(0)
print yellow("Highstate is still running.\nPolling again in {} seconds.\n".format(timeout))
time.sleep(timeout)
salt_run(method='jobs.active', stdout=result)
print green("Highstate complete.\n")
result.close()
def _stash_changes(self):
self.stashed_changes = False
result = self._git_status()
if 'nothing to commit, working directory clean' not in result:
print(yellow("Stashing changes"))
self.repository.git.stash()
self.stashed_changes = True
def highstate_complete():
"""
Poll jobs active waiting for it to become empty after an unattended_highstate was started
"""
timeout = 15
result = StringIO()
salt_run(method='jobs.active', stdout=result)
while len(result.getvalue().strip()):
result.truncate(0)
print yellow(
"Highstate is still running.\nPolling again in {} seconds.\n".
format(timeout))
time.sleep(timeout)
salt_run(method='jobs.active', stdout=result)
print green("Highstate complete.\n")
result.close()
def refresh_pillars(selector="'*'", prefix='', salt_environment=None):
"""
Wrapper for clear_cache and reload_pillar
:param selector: String selector
:param prefix: String used for compound or grain matches
:param salt_environment: string representing our target environment
:return:
"""
print yellow("Clearing pillar cache")
clear_cache(
selector=selector,
prefix=prefix,
salt_environment=salt_environment
)
print yellow("Refreshing pillar data")
reload_pillar(
selector=selector,
prefix=prefix,
salt_environment=salt_environment
)
def get_rendered_pillar_location(
pillar_dir=None,
projects_location=None,
parse_top_sls=True,
target=None
):
"""
Returns path to rendered pillar.
Use to render pillars written in jinja locally not to upload unwanted data to network.
i.e. you can use constructs like:
{% include 'opg-lpa-dev/pillar/services.sls' %}
If you want salt to later render pillars with grain context use constructs like:
{% raw %} {{grains.get('roles')}} {% endraw %}
{{" {{grains.get('roles')}} "}}
To allow for server side templating of top.sls, you will need set: `parse_top_sls=False`
In case there is no top.sls in pillar root than it returns: None
"""
if 'use_project_dir' not in 'env':
env.use_project_dir = False
if projects_location is None:
projects_location = _get_projects_location()
pillars = __load_pillar_dirs(pillar_dir, projects_location, target)
jinja_env = _set_template_env(pillars, projects_location)
dest_location = tempfile.mkdtemp()
if parse_top_sls:
files_to_render = _pillar_from_top_sls(dest_location, jinja_env)
elif len(pillars):
files_to_render, output_dir = _pillar_from_dirs(pillars)
else:
print(yellow("No template files where found to render"))
files_to_render = []
if __render_templates(files_to_render, dest_location, jinja_env) is False:
print(red("Aborting due to pillar failing to render"))
exit(-1)
return dest_location
def get_rendered_pillar_location(pillar_dir=None,
projects_location=None,
parse_top_sls=True,
target=None):
"""
Returns path to rendered pillar.
Use to render pillars written in jinja locally not to upload unwanted data to network.
i.e. you can use constructs like:
{% include 'opg-lpa-dev/pillar/services.sls' %}
If you want salt to later render pillars with grain context use constructs like:
{% raw %} {{grains.get('roles')}} {% endraw %}
{{" {{grains.get('roles')}} "}}
To allow for server side templating of top.sls, you will need set: `parse_top_sls=False`
In case there is no top.sls in pillar root than it returns: None
"""
if 'use_project_dir' not in 'env':
env.use_project_dir = False
if projects_location is None:
projects_location = _get_projects_location()
pillars = __load_pillar_dirs(pillar_dir, projects_location, target)
jinja_env = _set_template_env(pillars, projects_location)
dest_location = tempfile.mkdtemp()
if parse_top_sls:
files_to_render = _pillar_from_top_sls(dest_location, jinja_env)
elif len(pillars):
files_to_render, output_dir = _pillar_from_dirs(pillars)
else:
print(yellow("No template files where found to render"))
files_to_render = []
if __render_templates(files_to_render, dest_location, jinja_env) is False:
print(red("Aborting due to pillar failing to render"))
exit(-1)
return dest_location
def commit_change_set(self):
files_exist = []
for change in self.change_set:
if change in self._git_status():
files_exist.append(change)
if len(files_exist):
try:
self._stash_changes()
self._checkout_branch()
self._pull_branch()
self._pop_changes()
self._git_commit(files_exist, self.message)
except GitCommandError as e:
if not e.stderr.contains('No stash found.'):
print(red("Git returned: {}".format(e.stderr)))
exit(e.status)
else:
print(yellow(self.NO_CHANGES_TO_COMMIT))
return self.NO_CHANGES_TO_COMMIT
def filter(self, **kwargs):
"""
return: list of objects matching filter args
typically provide should support filter 'name'='foo'
"""
instances = []
if 'name' in kwargs:
name = kwargs['name']
zone_config = get_provider_zone_config()
assert zone_config['driver'] == 'static'
for host_spec in zone_config['hosts']:
if host_spec['name'] == name:
print("selected static instance: {}".format(host_spec['name']))
instances.append(host_spec)
break
else:
print(yellow("Warning: {} not found!".format(name), bold=True))
else:
raise NotImplementedError()
return instances
def filter(self, **kwargs):
"""
return: list of objects matching filter args
typically provide should support filter 'name'='foo'
"""
instances = []
if 'name' in kwargs:
name = kwargs['name']
selected = False
for reservation in self.connection.get_all_instances():
instance = reservation.instances[0]
if instance.state not in ['terminated', 'shutting-down']:
if "Name" in instance.tags and instance.tags["Name"] == name:
instances.append(instance)
selected = True
print(green("Selected aws instance: {}".format(instance.id)))
if not selected:
print(yellow("Warning: {} not found!".format(name), bold=True))
else:
raise NotImplementedError()
return instances
def filter(self, **kwargs):
"""
return: list of objects matching filter args
typically provide should support filter 'name'='foo'
"""
instances = []
if 'name' in kwargs:
name = kwargs['name']
zone_config = get_provider_zone_config()
assert zone_config['driver'] == 'static'
for host_spec in zone_config['hosts']:
if host_spec['name'] == name:
print("selected static instance: {}".format(
host_spec['name']))
instances.append(host_spec)
break
else:
print(yellow("Warning: {} not found!".format(name), bold=True))
else:
raise NotImplementedError()
return instances
def build_salt_dirs():
import yaml
"""
get list of pillar directories for cotton
:return: dict with list of dirs or error
"""
print(yellow("{}".format('Building pillar directory list')))
try:
dir_list = []
not_found = []
root_list = []
with open('sources.yml') as f:
_dirs = yaml.safe_load(f)
pillar_dirs = _dirs['pillar'][
env.provider_zone] + _dirs['pillar']['common']
pillar_roots = _dirs['pillar_roots'][
env.provider_zone] + _dirs['pillar_roots']['common']
for pillar_dir_name in pillar_dirs:
if os.path.exists(pillar_dir_name):
dir_list.append(pillar_dir_name)
else:
not_found.append(pillar_dir_name)
for pillar_rootdir_name in pillar_roots:
if os.path.exists(pillar_rootdir_name):
root_list.append(pillar_rootdir_name)
else:
not_found.append(pillar_rootdir_name)
return {
'pillar_dir': dir_list,
'pillar_root': root_list,
'missing': not_found
}
except (OSError, IOError):
return {'error': 'Failed to open sources.yml'}
def _pop_changes(self):
if self.stashed_changes:
print(yellow("Popping changes"))
self.repository.git.stash('pop')
def _checkout_branch(self):
print(yellow("Checking out {}".format(self.target_branch)))
self.repository.git.checkout(self.target_branch)
def get_rendered_pillar_location():
"""
Returns path to rendered pillar.
Use to render pillars written in jinja locally not to upload unwanted data to network.
i.e. you can use constructs like:
{% include 'opg-lpa-dev/pillar/services.sls' %}
In case there is no top.sls in pillar root than it returns: None
"""
from jinja2 import Environment
from jinja2 import FileSystemLoader
from jinja2.exceptions import TemplateNotFound
assert env.project
projects_location = _get_projects_location()
jinja_env = Environment(loader=FileSystemLoader([
os.path.join(projects_location, env.project, 'pillar'),
projects_location
]))
# let's get rendered top.sls for configured project
try:
top_sls = jinja_env.get_template('top.sls').render(env=env)
except TemplateNotFound:
print(red("Missing top.sls in pillar location. Skipping rendering."))
return None
top_content = yaml.load(top_sls)
dest_location = tempfile.mkdtemp()
with open(os.path.join(dest_location, 'top.sls'), 'w') as f:
f.write(top_sls)
# get list of files referenced by top.sls
files_to_render = []
for k0, v0 in top_content.iteritems():
for k1, v1 in v0.iteritems():
for file_short in v1:
# We force this file to be relative in case jinja failed rendering
# a variable. This would make the filename start with / and instead of
# writing under dest_location it will try to write in /
files_to_render.append('./' + file_short.replace('.', '/') +
'.sls')
# render and save templates
for template_file in files_to_render:
filename = os.path.abspath(os.path.join(dest_location, template_file))
print(
yellow("Pillar template_file: {} --> {}".format(
template_file, filename)))
if not os.path.isdir(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
try:
template_rendered = jinja_env.get_template(template_file).render(
env=env)
except TemplateNotFound:
template_rendered = ''
print(
yellow("Pillar template_file not found: {} --> {}".format(
template_file, filename)))
with open(os.path.join(dest_location, template_file), 'w') as f:
f.write(template_rendered)
print(green("Pillar was rendered in: {}".format(dest_location)))
return dest_location
def _pull_branch(self):
print(yellow("Rebasing against branch"))
self.repository.git.pull('--rebase')
def _reset_uncommitted_files(self):
print(yellow("Resetting uncommitted files"))
self.repository.git.checkout('.')
def _clean_unstaged_files(self):
print(yellow("Cleaning up unstaged files"))
for untracked_file in self.repository.untracked_files:
os.unlink(untracked_file)
