def _compare_config(self):
"""Smart-ish comparison of Spotinst config to our own.
This method is called by the EnsurableBaseClass to compare the desired
(local) config with the existing (remote) config of the ElastiGroup.
A simple == comparison will not work because there are additional
fields returned by the Spotinst API (id, createdAt, updatedAt, and
more) that will never be in the desired configuration object.
This method makes copies of the configuration objects, strips out the
fields that we cannot compare against, and then diffs. If a diff is
detected, it logs out the diff for the end user, and then returns
False.
Returns:
True: the configs match
False: the configs do not match
"""
# For the purpose of comparing the two configuration dicts, we need to
# modify them (below).. so first lets copy them so we don't modify the
# originals.
new = copy.deepcopy(self._config)
existing = copy.deepcopy(self._group)
# If existing is none, then return .. there is no point in diffing the
# config if the group doesn't exist! Note, this really only happens in
# a dry run where we're creating the group because the group
if existing is None:
raise gen.Return(True)
# Strip out some of the Spotinst generated and managed fields that
# should never end up in either our new or existing configs.
for field in ('id', 'createdAt', 'updatedAt', 'userData'):
for g in (new, existing):
g['group'].pop(field, None)
# Decode both of the userData fields so we can actually see the
# userdata differences.
for config in (new, existing):
config['group']['compute']['launchSpecification']['userData'] = (
base64.b64decode(config['group']['compute']
['launchSpecification']['userData']))
# We only allow a user to supply a single subnetId for each AZ (this is
# handled by the ElastiGroupSchema). Spotinst returns back though both
# the original setting, as well as a list of subnetIds. We purge that
# from our comparison here.
for az in existing['group']['compute']['availabilityZones']:
az.pop('subnetIds', None)
diff = utils.diff_dicts(existing, new)
if diff:
self.log.warning('Group configurations do not match')
for line in diff.split('\n'):
self.log.info('Diff: %s' % line)
return False
return True
def _compare_config(self):
"""Smart-ish comparison of Spotinst config to our own.
This method is called by the EnsurableBaseClass to compare the desired
(local) config with the existing (remote) config of the ElastiGroup.
A simple == comparison will not work because there are additional
fields returned by the Spotinst API (id, createdAt, updatedAt, and
more) that will never be in the desired configuration object.
This method makes copies of the configuration objects, strips out the
fields that we cannot compare against, and then diffs. If a diff is
detected, it logs out the diff for the end user, and then returns
False.
Returns:
True: the configs match
False: the configs do not match
"""
# For the purpose of comparing the two configuration dicts, we need to
# modify them (below).. so first lets copy them so we don't modify the
# originals.
new = copy.deepcopy(self._config)
existing = copy.deepcopy(self._group)
# If existing is none, then return .. there is no point in diffing the
# config if the group doesn't exist! Note, this really only happens in
# a dry run where we're creating the group because the group
if existing is None:
raise gen.Return(True)
# Strip out some of the Spotinst generated and managed fields that
# should never end up in either our new or existing configs.
for field in ('id', 'createdAt', 'updatedAt', 'userData'):
for g in (new, existing):
g['group'].pop(field, None)
# Decode both of the userData fields so we can actually see the
# userdata differences.
for config in (new, existing):
config['group']['compute']['launchSpecification']['userData'] = (
base64.b64decode(config['group']['compute']
['launchSpecification']['userData']))
# We only allow a user to supply a single subnetId for each AZ (this is
# handled by the ElastiGroupSchema). Spotinst returns back though both
# the original setting, as well as a list of subnetIds. We purge that
# from our comparison here.
for az in existing['group']['compute']['availabilityZones']:
az.pop('subnetIds', None)
diff = utils.diff_dicts(existing, new)
if diff:
self.log.warning('Group configurations do not match')
for line in diff.split('\n'):
self.log.info('Diff: %s' % line)
return False
return True
def _diff_params_safely(self, remote, local):
"""Safely diffs the CloudFormation parameters.
Does a comparison of the locally supplied parameters, and the remotely
discovered (already set) CloudFormation parameters. When they are
different, shows a clean diff and returns False.
Takes into account NoEcho parameters which cannot be diff'd, so should
not be included in the output (likely because they are passwords).
Args:
Remote: A list of objects, each having a ParameterKey and
ParameterValue.
Local: A list of objects, each having a ParameterKey and
ParameterValue.
Returns:
Boolean
"""
# If there are any NoEcho parameters, we can't diff them .. Amazon
# returns them as *****'s and we're unable to compare them. Also, we
# wouldn't want to print these out in our logs because they're almost
# certainly passwords. Therefore, we should simply skip them in the
# diff.
for p in self._noecho_params:
self.log.debug(
'Removing "%s" from parameters before comparison.' % p)
remote = [pair for pair in remote if pair['ParameterKey'] != p]
local = [pair for pair in local if pair['ParameterKey'] != p]
# Remove any resolved parameter values that were inserted by SSM
# so that only supplied parameter values are compared.
filtered_remote = []
for param in remote:
filtered_param = {}
for k, v in param.items():
if k != "ResolvedValue":
filtered_param[k] = v
filtered_remote.append(filtered_param)
remote = filtered_remote
diff = utils.diff_dicts(remote, local)
if diff:
self.log.warning('Stack parameters do not match.')
for line in diff.split('\n'):
self.log.info('Diff: %s' % line)
return True
return False
def _diff_params_safely(self, remote, local):
"""Safely diffs the CloudFormation parameters.
Does a comparison of the locally supplied parameters, and the remotely
discovered (already set) CloudFormation parameters. When they are
different, shows a clean diff and returns False.
Takes into account NoEcho parameters which cannot be diff'd, so should
not be included in the output (likely because they are passwords).
Args:
Remote: A list of objects, each having a ParameterKey and
ParameterValue.
Local: A list of objects, each having a ParameterKey and
ParameterValue.
Returns:
Boolean
"""
# If there are any NoEcho parameters, we can't diff them .. Amazon
# returns them as *****'s and we're unable to compare them. Also, we
# wouldn't want to print these out in our logs because they're almost
# certainly passwords. Therefore, we should simply skip them in the
# diff.
for p in self._noecho_params:
self.log.debug('Removing "%s" from parameters before comparison.' %
p)
remote = [pair for pair in remote if pair['ParameterKey'] != p]
local = [pair for pair in local if pair['ParameterKey'] != p]
# Remove any resolved parameter values that were inserted by SSM
# so that only supplied parameter values are compared.
filtered_remote = []
for param in remote:
filtered_param = {}
for k, v in param.items():
if k != "ResolvedValue":
filtered_param[k] = v
filtered_remote.append(filtered_param)
remote = filtered_remote
diff = utils.diff_dicts(remote, local)
if diff:
self.log.warning('Stack parameters do not match.')
for line in diff.split('\n'):
self.log.info('Diff: %s' % line)
return True
return False
def _ensure_inline_policies(self, name):
"""Ensures that all of the inline IAM policies for a entity are managed
This method has three stages.. first it ensures that any missing
policies (as determined by the policy name) are applied to a entity.
Second, it determines if any existing policies have changed locally and
need to be updated in IAM. Finally it purges unmanaged policies that
were applied to a entity out of band.
args:
name: The entity to manage
"""
# Get the list of current entity policies first
existing_policies = yield self._get_entity_policies(name)
# First, push any policies that we have listed, but aren't in the
# entity
tasks = []
for policy in (set(self.inline_policies.keys()) -
set(existing_policies.keys())):
policy_doc = self.inline_policies[policy]
tasks.append(self._put_entity_policy(name, policy, policy_doc))
yield tasks
# Do we have matching policies that we're managing here, and are
# already attached to the entity profile? Lets make sure each one of
# those matches the policy we have here, and update it if necessary.
tasks = []
for policy in (set(self.inline_policies.keys())
& set(existing_policies.keys())):
new = self.inline_policies[policy]
exist = existing_policies[policy]
diff = utils.diff_dicts(exist, new)
if diff:
self.log.info('Policy %s differs from Amazons:' % policy)
for line in diff.split('\n'):
self.log.info('Diff: %s' % line)
policy_doc = self.inline_policies[policy]
tasks.append(self._put_entity_policy(name, policy, policy_doc))
yield tasks
# Purge any policies we found in AWS that were not listed in our actor
tasks = []
for policy in (set(existing_policies.keys()) -
set(self.inline_policies.keys())):
tasks.append(self._delete_entity_policy(name, policy))
yield tasks
def _compare_tags(self):
new = self.option('tags')
if new is None:
self.log.debug('Not managing Tags')
raise gen.Return(True)
exist = yield self._get_tags()
diff = utils.diff_dicts(exist, new)
if not diff:
self.log.debug('Bucket tags match')
raise gen.Return(True)
self.log.info('Bucket tags differs from Amazons:')
for line in diff.split('\n'):
self.log.info('Diff: %s' % line)
raise gen.Return(False)
def _compare_tags(self):
new = self.option('tags')
if new is None:
self.log.debug('Not managing Tags')
raise gen.Return(True)
exist = yield self._get_tags()
diff = utils.diff_dicts(exist, new)
if not diff:
self.log.debug('Bucket tags match')
raise gen.Return(True)
self.log.info('Bucket tags differs from Amazons:')
for line in diff.split('\n'):
self.log.info('Diff: %s' % line)
raise gen.Return(False)
def _compare_policy(self):
new = self.policy
if self.policy is None:
self.log.debug('Not managing policy')
raise gen.Return(True)
exist = yield self._get_policy()
# Now, diff our new policy from the existing policy. If there is no
# difference, then we bail out of the method.
diff = utils.diff_dicts(exist, new)
if not diff:
self.log.debug('Bucket policy matches')
raise gen.Return(True)
# Now, print out the diff..
self.log.info('Bucket policy differs from Amazons:')
for line in diff.split('\n'):
self.log.info('Diff: %s' % line)
raise gen.Return(False)
def _compare_lifecycle(self):
existing = yield self._get_lifecycle()
new = self.lifecycle
if new is None:
self.log.debug('Not managing lifecycle')
raise gen.Return(True)
# Now sort through the existing Lifecycle configuration and the one
# that we've built locally. If there are any differences, we're going
# to push an all new config.
diff = utils.diff_dicts(json.loads(jsonpickle.encode(existing)),
json.loads(jsonpickle.encode(new)))
if not diff:
raise gen.Return(True)
self.log.info('Lifecycle configurations do not match. Updating.')
for line in diff.split('\n'):
self.log.info('Diff: %s' % line)
raise gen.Return(False)
def _compare_policy(self):
new = self.policy
if self.policy is None:
self.log.debug('Not managing policy')
raise gen.Return(True)
exist = yield self._get_policy()
# Now, diff our new policy from the existing policy. If there is no
# difference, then we bail out of the method.
diff = utils.diff_dicts(exist, new)
if not diff:
self.log.debug('Bucket policy matches')
raise gen.Return(True)
# Now, print out the diff..
self.log.info('Bucket policy differs from Amazons:')
for line in diff.split('\n'):
self.log.info('Diff: %s' % line)
raise gen.Return(False)
def _ensure_assume_role_doc(self, name):
"""Ensures that the Assume Role Policy for a Role is up to date.
Downloads the existing Assume Role Policy for a given Role, then
compares it against our configured policy and optionally updates it if
they differ.
Args:
name: The role we're workin with
"""
# Get our existing role policy from the entity
entity = yield self._get_entity(name)
# If the entity doesn't exist, then we must be in a Dry run and the
# role hasn't been created yet. Just bail silently.
if not entity:
raise gen.Return()
# Parse the raw data into a dict we can compare
exist = self._policy_doc_to_dict(entity['assume_role_policy_document'])
new = self.assume_role_policy_doc
# Now diff it against our desired policy. If no diff, then quietly
# return.
diff = utils.diff_dicts(exist, new)
if not diff:
self.log.debug('Assume Role Policy documents match')
raise gen.Return()
self.log.info('Assume Role Policy differs from Amazons:')
for line in diff.split('\n'):
self.log.info('Diff: %s' % line)
if self._dry:
self.log.warning('Would have updated the Assume Role Policy Doc')
raise gen.Return()
self.log.info('Updating the Assume Role Policy Document')
yield self.thread(self.iam_conn.update_assume_role_policy, name,
json.dumps(new))
def _compare_lifecycle(self):
existing = yield self._get_lifecycle()
new = self.lifecycle
if new is None:
self.log.debug('Not managing lifecycle')
raise gen.Return(True)
# Now sort through the existing Lifecycle configuration and the one
# that we've built locally. If there are any differences, we're going
# to push an all new config.
diff = utils.diff_dicts(
json.loads(jsonpickle.encode(existing)),
json.loads(jsonpickle.encode(new)))
if not diff:
raise gen.Return(True)
self.log.info('Lifecycle configurations do not match. Updating.')
for line in diff.split('\n'):
self.log.info('Diff: %s' % line)
raise gen.Return(False)
def _ensure_template(self, stack):
"""Compares and updates the state of a CF Stack template
Compares the current template body against the template body for the
live running stack. If they're different. Triggers a Change Set
creation and ultimately executes the change set.
TODO: Support remote template_url comparison!
args:
stack: A Boto3 Stack object
"""
needs_update = False
# TODO: Implement this
if self._template_url:
self.log.warning('Cannot compare against remote template url')
raise gen.Return()
# Get the current template for the stack, and get our local template
# body. Make sure they're in the same form (dict).
existing = yield self._get_stack_template(stack['StackId'])
new = json.loads(self._template_body)
# Compare the two templates. If they differ at all, log it out for the
# user and flip the needs_update bit.
diff = utils.diff_dicts(existing, new)
if diff:
self.log.warning('Stack templates do not match.')
for line in diff.split('\n'):
self.log.info('Diff: %s' % line)
# Plan to make a change set!
needs_update = True
# Get and compare the parameters we have vs the ones in CF. If they're
# different, plan to do an update!
if self._diff_params_safely(
stack.get('Parameters', {}),
self._parameters):
needs_update = True
# If needs_update isn't set, then the templates are the same and we can
# bail!
if not needs_update:
self.log.debug('Stack matches configuration, no changes necessary')
raise gen.Return()
# If we're here, the templates have diverged. Generate the change set,
# log out the changes, and execute them.
change_set_req = yield self._create_change_set(stack)
change_set = yield self._wait_until_change_set_ready(
change_set_req['Id'], 'Status', 'CREATE_COMPLETE')
self._print_change_set(change_set)
# Ok run the change set itself!
try:
yield self._execute_change_set(
change_set_name=change_set_req['Id'])
except (ClientError, StackFailed) as e:
raise StackFailed(e)
# In dry mode, delete our change set so we don't leave it around as
# cruft. THis isn't necessary in the real run, because the changeset
# cannot be deleted once its been applied.
if self._dry:
yield self.thread(self.cf3_conn.delete_change_set,
ChangeSetName=change_set_req['Id'])
self.log.info('Done updating template')
def test_diff_dicts(self):
p1 = {'a': 'a', 'b': 'b'}
p2 = {'a': 'a', 'c': 'c'}
self.assertEqual(None, utils.diff_dicts(p1, p1))
self.assertNotEqual(None, utils.diff_dicts(p1, p2))
def _ensure_template(self, stack):
"""Compares and updates the state of a CF Stack template
Compares the current template body against the template body for the
live running stack. If they're different. Triggers a Change Set
creation and ultimately executes the change set.
TODO: Support remote template_url comparison!
args:
stack: A Boto3 Stack object
"""
needs_update = False
# TODO: Implement this
if self._template_url:
self.log.warning('Cannot compare against remote template url')
raise gen.Return()
# Get the current template for the stack, and get our local template
# body. Make sure they're in the same form (dict).
existing = yield self._get_stack_template(stack['StackId'])
new = json.loads(self._template_body)
# Compare the two templates. If they differ at all, log it out for the
# user and flip the needs_update bit.
diff = utils.diff_dicts(existing, new)
if diff:
self.log.warning('Stack templates do not match.')
for line in diff.split('\n'):
self.log.info('Diff: %s' % line)
# Plan to make a change set!
needs_update = True
# Get and compare the parameters we have vs the ones in CF. If they're
# different, plan to do an update!
if self._diff_params_safely(
stack.get('Parameters', []),
self._parameters):
needs_update = True
# If needs_update isn't set, then the templates are the same and we can
# bail!
if not needs_update:
self.log.debug('Stack matches configuration, no changes necessary')
raise gen.Return()
# If we're here, the templates have diverged. Generate the change set,
# log out the changes, and execute them.
change_set_req = yield self._create_change_set(stack)
change_set = yield self._wait_until_change_set_ready(
change_set_req['Id'], 'Status', 'CREATE_COMPLETE')
self._print_change_set(change_set)
# Ok run the change set itself!
try:
yield self._execute_change_set(
change_set_name=change_set_req['Id'])
except (ClientError, StackFailed) as e:
raise StackFailed(e)
# In dry mode, delete our change set so we don't leave it around as
# cruft. THis isn't necessary in the real run, because the changeset
# cannot be deleted once its been applied.
if self._dry:
yield self.api_call(self.cf3_conn.delete_change_set,
ChangeSetName=change_set_req['Id'])
self.log.info('Done updating template')
def test_diff_dicts(self):
p1 = {'a': 'a', 'b': 'b'}
p2 = {'a': 'a', 'c': 'c'}
self.assertEquals(None, utils.diff_dicts(p1, p1))
self.assertNotEquals(None, utils.diff_dicts(p1, p2))
