def is_state_definition_equivalent(self):
"""
Checks if the current state definition and desired state definitions are equivalent including
attributes.
Returns:
bool
"""
filtered_current_state_definition = pcf_util.param_filter(
self.current_state_definition,
BatchComputeEnvironment.UPDATE_PARAM_FILTER)
filtered_desired_state_definition = pcf_util.param_filter(
self.desired_state_definition,
BatchComputeEnvironment.UPDATE_PARAM_FILTER)
diff = pcf_util.diff_dict(filtered_desired_state_definition,
filtered_current_state_definition)
# TODO might need more comp logic here
if not self.desired_state_definition[
"serviceRole"] in self.current_state_definition["serviceRole"]:
return False
if diff or len(diff) != 0:
return False
return True
def is_state_definition_equivalent(self):
"""
Determines if the current state definition matches the desired state definition.
:return: bool
"""
self.sync_state()
filtered_current_state_definition = pcf_util.param_filter(
self.current_state_definition, RDS.UPDATE_PARAM_FILTER)
filtered_desired_state_definition = pcf_util.param_filter(
self.desired_state_definition, RDS.UPDATE_PARAM_FILTER)
filtered_current_state_definition[
'ApplyImmediately'] = self.desired_state_definition[
'ApplyImmediately']
filtered_current_state_definition[
'SkipFinalSnapshot'] = self.desired_state_definition[
'SkipFinalSnapshot']
diff = pcf_util.diff_dict(filtered_current_state_definition,
filtered_desired_state_definition)
if not diff or len(diff) == 0:
return True
else:
logger.debug(
"State is not equivalent for {0} with diff: {1}".format(
self.get_pcf_id(), json.dumps(diff)))
return False
def is_state_definition_equivalent(self):
"""
Compared the desired state and current state definition
Returns:
bool
"""
self.sync_state()
# use filters to remove any extra information
self.current_state_definition = {
key: self.current_state_definition[key]
for key in SQSQueue.DEFINITION_FILTER
if key in self.current_state_definition.keys()
}
self.desired_state_definition = {
key: self.desired_state_definition[key]
for key in SQSQueue.DEFINITION_FILTER
if key in self.desired_state_definition.keys()
}
self.desired_state_definition["Attributes"] = {
key: self.desired_state_definition.get("Attributes")[key]
for key in SQSQueue.START_ATTR
if key in self.desired_state_definition.get("Attributes").keys()
}
# only compare attributes specified in desired, ignore all else
self.current_state_definition["Attributes"] = {
key: self.current_state_definition.get("Attributes")[key]
for key in self.desired_state_definition.get("Attributes").keys()
if key in self.current_state_definition.get("Attributes").keys()
}
diff_dict = pcf_util.diff_dict(self.current_state_definition,
self.desired_state_definition)
return diff_dict == {}
def test_diff_dict():
orig = {'test': 72, 'test2': 66}
update = {
'test': 72,
}
diff = pcf_util.diff_dict(orig, update)
assert (diff == {})
def _update(self):
# only updates InstanceCount
desired_instances = self.desired_state_definition.get("Instances").get(
"InstanceGroups")
# instance pool
if not desired_instances:
desired_instances = self.desired_state_definition.get(
"Instances").get("InstanceFleets")
# TODO neither pool or group
if not desired_instances:
return True
desired_instance_dict = pcf_util.list_to_dict("Name",
desired_instances)
desired_instance_dict = {
k: pcf_util.param_filter(v, EMRCluster.FILTERED_UPDATE_PARAMS)
for k, v in desired_instance_dict.items()
}
current_instance_dict = {
k:
pcf_util.keep_and_replace_keys(v,
EMRCluster.UPDATE_PARAM_CONVERSIONS)
for k, v in self.current_state_definition.get("Instances").items()
}
diff = pcf_util.diff_dict(current_instance_dict, desired_instance_dict)
for k, v in diff.items():
curr_instance = self.current_state_definition["Instances"].get(
k, {}).get('Id')
#instance group
if curr_instance and self.current_state_definition[
"Instances"].get(k, {}).get('InstanceGroupType'):
self.client.modify_instance_groups(
ClusterId=self._get_cluster_id(),
InstanceGroups=[{
"InstanceGroupId":
curr_instance,
"InstanceCount":
v["InstanceCount"]["updated"]
}])
#instance pool
elif curr_instance:
od_cap = v.get("TargetOnDemandCapacity", {}).get("updated", 0)
spot_cap = v.get("TargetSpotCapacity", {}).get("updated", 0)
self.client.modify_instance_fleet(
ClusterId=self._get_cluster_id(),
InstanceFleet={
"InstanceFleetId": curr_instance,
"TargetOnDemandCapacity": od_cap,
"TargetSpotCapacity": spot_cap
})
def is_state_definition_equivalent(self):
"""
Compare current and desired state definition
Returns:
bool
"""
self.sync_state()
diff_dict = pcf_util.diff_dict(pcf_util.param_filter(self.current_state_definition, ECRRepository.DEFINITION_FILTER),
pcf_util.param_filter(self.desired_state_definition, ECRRepository.DEFINITION_FILTER))
return diff_dict == {}
def is_state_definition_equivalent(self):
"""
Compared the desired state and current state definition
Returns:
bool
"""
self.current_state_definition = pcf_util.param_filter(
self.current_state_definition, IAMRole.START_PARAMS_FILTER)
self.current_state_definition['custom_config'] = {}
self.get_iam_policies()
diff_dict = {}
if self.desired_state != State.terminated:
#Comparing currently attached policies to desired policies
try:
attached_policy_arns = self.client.list_attached_role_policies(
RoleName=self.role_name,
PathPrefix=self.desired_state_definition.get('Path', '/'))
except ClientError as e:
#No attached policies
attached_policy_arns = {}
if attached_policy_arns.get('AttachedPolicies'):
current_policy_arns = [
p.get('PolicyArn')
for p in attached_policy_arns.get('AttachedPolicies')
]
self.current_state_definition['custom_config'][
'policy_arns'] = current_policy_arns
else:
self.current_state_definition['custom_config'][
'policy_arns'] = []
if isinstance(
self.desired_state_definition.get(
'AssumeRolePolicyDocument'), str):
self.desired_state_definition[
'AssumeRolePolicyDocument'] = json.loads(
self.desired_state_definition.get(
'AssumeRolePolicyDocument'))
self.current_state_definition['custom_config'][
'IsInstanceProfile'] = self.custom_config.get(
'IsInstanceProfile', False)
diff_dict = pcf_util.diff_dict(self.current_state_definition,
self.desired_state_definition)
return diff_dict == {}
def is_state_definition_equivalent(self):
"""
Compared the desired state and current state definition
Returns:
bool
"""
self.get_state()
self.current_state_definition = pcf_util.param_filter(
self.current_state_definition, CloudWatchEvent.START_PARAM_FILTER)
desired_definition = pcf_util.param_filter(
self.desired_state_definition, CloudWatchEvent.START_PARAM_FILTER)
diff_dict = pcf_util.diff_dict(self.current_state_definition,
desired_definition)
return diff_dict == {}
def is_state_definition_equivalent(self):
"""
Compared the desired state and current state definition
Returns:
bool
"""
self.sync_state()
# remove extra information
self.current_state_definition = pcf_util.param_filter(self.current_state_definition, CloudWatchLog.START_PARAM_FILTER)
# get the tags and add them on (they're together when creating, but separate when retrieving)
self.current_state_definition["tags"] = self.client.list_tags_log_group(logGroupName=
self.group_name).get("tags")
self.desired_state_definition = pcf_util.param_filter(self.desired_state_definition, CloudWatchLog.START_PARAM_FILTER)
diff_dict = pcf_util.diff_dict(self.current_state_definition, self.desired_state_definition)
return diff_dict == {}
def is_state_definition_equivalent(self):
"""
Determines if the current state definition and the desired state definition are equivalent.
Returns:
bool
"""
self.sync_state()
filtered_curr_def = {
key: self.current_state_definition[key]
for key in ElasticLoadBalancing.UPDATE_PARAM_FILTER
if key in self.current_state_definition.keys()
}
filtered_des_def = {
key: self.desired_state_definition[key]
for key in self.desired_state_definition.keys()
if key not in ElasticLoadBalancing.REMOVE_PARAM_FILTER
}
def_diff = pcf_util.diff_dict(filtered_curr_def, filtered_des_def)
curr_tags = self.client.describe_tags(LoadBalancerNames=[
self.elb_name,
])
curr_tags = curr_tags.get('TagDescriptions')[0].get('Tags')
des_tags = filtered_des_def.get('Tags')
curr_listeners = [
x['Listener']
for x in filtered_curr_def.get('ListenerDescriptions')
]
des_listeners = filtered_des_def['Listeners']
for item in des_listeners:
item.update({'InstanceProtocol': item.get('Protocol').upper()})
item.update({'Protocol': item.get('Protocol').upper()})
def_diff.pop('Listeners', None)
def_diff.pop('Tags', None)
if def_diff or _need_update(curr_listeners,
des_listeners) or _need_update(
curr_tags, des_tags):
return False
return True
def is_state_definition_equivalent(self):
"""
Determines if the current state definition and the desired state definition are equivalent.
Returns:
bool
"""
filtered_curr_def = pcf_util.param_filter(
self.current_state_definition, self.UPDATE_PARAM_FILTER)
filtered_des_def = pcf_util.param_filter(self.desired_state_definition,
self.REMOVE_PARAM_FILTER,
True)
def_diff = pcf_util.diff_dict(filtered_curr_def, filtered_des_def)
curr_tags = self.client.describe_tags(ResourceArns=[self.arn])
curr_tags = curr_tags.get('TagDescriptions')[0].get('Tags', None)
des_tags = self.desired_state_definition.get('Tags', None)
curr_availabilty_zones = pcf_util.find_nested_dict_value(
self.current_state_definition, ['AvailabilityZones'])
curr_subnets_list = pcf_util.transform_list_of_dicts_to_desired_list(
curr_availabilty_zones, 'SubnetId')
des_subnets_list = self.desired_state_definition.get('Subnets')
curr_listeners = [
pcf_util.param_filter(x, {'SslPolicy'}, True)
for x in self.get_listener_status()
]
des_listeners = self.custom_config['Listeners']
for item in des_listeners:
item.update({'Protocol': item.get('Protocol').upper()})
for item in curr_listeners:
if 'ListenerArn' in item:
item.pop('ListenerArn')
if def_diff or self._need_update(curr_tags, des_tags) or self._need_update(curr_listeners, des_listeners) \
or not pcf_util.is_list_equal(curr_subnets_list, des_subnets_list):
return False
return True
def is_state_definition_equivalent(self):
"""
Checks if the current state definition and desired state definitions are equivalent including
attributes.
Returns:
bool
"""
filtered_desired_def = pcf_util.param_filter(
self.desired_state_definition, self.UPDATE_PARAM_FILTER)
filtered_current_def = pcf_util.param_filter(
self.current_state_definition, self.UPDATE_PARAM_FILTER)
diff = pcf_util.diff_dict(filtered_desired_def, filtered_current_def)
if diff or len(diff) != 0:
return False
return True
def is_state_definition_equivalent(self):
"""
Compares the desired state and current state definitions.
Returns:
bool
"""
self.current_state_definition = pcf_util.param_filter(
self.current_state_definition, CloudFormationStack.PARAM_FILTER)
#seperate boto call to get the cloudformation template
response = self.client.get_template(StackName=self.stack_name,
TemplateStage="Original")
self.current_state_definition['TemplateBody'] = json.dumps(
response.get('TemplateBody'))
diff_dict = pcf_util.diff_dict(self.current_state_definition,
self.desired_state_definition)
return diff_dict == {}
def is_state_definition_equivalent(self):
"""
Determines if the current state definition matches the current state definition. Uses keep and remove function from pcf util to remove extra params in desired state that are not
in the current state.
Returns:
bool
"""
self.get_state()
desired_definition = pcf_util.param_filter(
self.desired_state_definition,
AutoScalingGroup.REMOVE_PARAM_FILTER, True)
diff = pcf_util.diff_dict(self.current_state_definition,
desired_definition)
if not diff or len(diff) == 0:
return True
else:
logger.debug(
"State is not equivalent for {0} with diff: {1}".format(
self.get_pcf_id(), json.dumps(diff)))
return False
def is_state_definition_equivalent(self):
"""
Compares the desired state and current state definitions.
Returns:
bool
"""
self.current_state_definition = pcf_util.param_filter(
self.current_state_definition, CloudFormationStack.PARAM_FILTER)
#seperate boto call to get the cloudformation template
response = self.client.get_template(StackName=self.stack_name,
TemplateStage="Original")
# template comes back with extra quotes and extra escape \ for new lines. not sure why
self.current_state_definition['TemplateBody'] = json.dumps(
response.get('TemplateBody')).replace('\\n', '\n')[1:-1]
filtered_desired_def = pcf_util.param_filter(
self.desired_state_definition, CloudFormationStack.PARAM_FILTER)
diff_dict = pcf_util.diff_dict(self.current_state_definition,
filtered_desired_def)
return diff_dict == {}
def is_state_definition_equivalent(self):
"""
Determines if state definitions are equivalent
Returns:
bool
"""
self.get_state()
diff = pcf_util.diff_dict(self.current_state_definition,
self.get_desired_state_definition())
if not diff or len(diff) == 0:
return True
else:
# can't json dump function
if diff.get('callbacks'):
diff['callbacks'] = {'new': '<function>'}
logger.debug(
"State is not equivalent for {0} with diff: {1}".format(
self.get_pcf_id(), json.dumps(diff)))
return False
def is_state_definition_equivalent(self):
"""
Determines if current state is equivalent to the desired state.
Returns:
bool
"""
desired_instances = self.desired_state_definition.get("Instances").get(
"InstanceGroups")
# instance pool
if not desired_instances:
desired_instances = self.desired_state_definition.get(
"Instances").get("InstanceFleets")
# TODO neither pool or group
if not desired_instances:
return True
desired_instance_dict = pcf_util.list_to_dict("Name",
desired_instances)
desired_instance_dict = {
k: pcf_util.param_filter(v, EMRCluster.FILTERED_UPDATE_PARAMS)
for k, v in desired_instance_dict.items()
}
current_instance_dict = {
k:
pcf_util.keep_and_replace_keys(v,
EMRCluster.UPDATE_PARAM_CONVERSIONS)
for k, v in self.current_state_definition.get("Instances").items()
}
# moto bug with instance fleets
if not current_instance_dict:
return True
diff = pcf_util.diff_dict(current_instance_dict, desired_instance_dict)
return diff == {}
def is_state_definition_equivalent(self):
"""
Determines if the current state definition matches the current state definition. Uses keep and remove function from pcf util to remove extra params in desired state that are not
in the current state.
:return: bool
"""
self.get_state()
diff = pcf_util.diff_dict(
self.current_state_definition,
pcf_util.keep_and_remove_keys(
self.get_desired_state_definition(),
Route53Record.REMOVE_PARAM_CONVERSIONS))
if not diff or len(diff) == 0:
return True
elif self.state == State.terminated:
return True
else:
logger.debug(
"State is not equivalent for {0} with diff: {1}".format(
self.get_pcf_id(), json.dumps(diff)))
return False
def is_state_definition_equivalent(self):
"""
Determines if state definitions are equivalent including list of tags.
Returns:
Bool
"""
if len(self.current_state_definition.get('Tags', [])) != len(
self.tags):
return False
tag_zip = zip(
sorted(self.current_state_definition.get('Tags', []),
key=lambda k: k["Value"]),
sorted(self.tags, key=lambda k: k["Value"]))
diff_tags = any(x != y for x, y in tag_zip)
if diff_tags:
return False
filtered_current_definition = pcf_util.param_filter(
self.get_current_state_definition(),
EBSVolume.UPDATE_PARAM_FILTER,
)
filtered_desired_definition = pcf_util.param_filter(
self.get_desired_state_definition(),
EBSVolume.UPDATE_PARAM_FILTER,
)
diff = pcf_util.diff_dict(filtered_current_definition,
filtered_desired_definition)
if diff:
return False
return True
def _update(self):
"""
Calls boto3 modify_volume(). If tags are changes then also calls delete_tags() and then create_tags()
Returns:
boto3 modify_volume() response
"""
filtered_current_definition = pcf_util.param_filter(
self.get_current_state_definition(),
EBSVolume.UPDATE_PARAM_FILTER,
)
filtered_desired_definition = pcf_util.param_filter(
self.get_desired_state_definition(),
EBSVolume.UPDATE_PARAM_FILTER,
)
diff = pcf_util.diff_dict(filtered_current_definition,
filtered_desired_definition)
# update tags
tag_zip = zip(
sorted(self.current_state_definition.get('Tags', []),
key=lambda k: k["Value"]),
sorted(self.tags, key=lambda k: k["Value"]))
diff_tags = any(x != y for x, y in tag_zip)
if diff_tags or len(self.current_state_definition.get(
'Tags', [])) != len(self.tags):
self.client.delete_tags(Resources=[self.volume_id])
self.client.create_tags(Resources=[self.volume_id], Tags=self.tags)
if not diff:
return
return self.client.modify_volume(VolumeId=self.volume_id,
**filtered_desired_definition)
def _update(self):
"""
updates the alb particle to match the desired state
"""
filtered_curr_def = pcf_util.param_filter(
self.current_state_definition, self.UPDATE_PARAM_FILTER)
filtered_des_def = pcf_util.param_filter(self.desired_state_definition,
self.REMOVE_PARAM_FILTER,
True)
diff = pcf_util.diff_dict(filtered_curr_def, filtered_des_def)
curr_tags = self.client.describe_tags(ResourceArns=[self.arn])
curr_tags = curr_tags.get('TagDescriptions')[0].get('Tags')
des_tags = self.desired_state_definition.get('Tags', [{}])
curr_listeners = [x for x in self.get_listener_status()]
# Format current listener to compare.
filtered_curr_listeners = [x for x in self.get_listener_status()]
for item in filtered_curr_listeners:
if 'ListenerArn' in item:
item.pop('ListenerArn')
des_listeners = self.custom_config['Listeners']
if diff.get('SecurityGroups', None):
self.client.set_security_groups(
LoadBalancerArn=self.arn,
SecurityGroups=filtered_des_def.get('SecurityGroups'))
curr_availabilty_zones = pcf_util.find_nested_dict_value(
self.current_state_definition, ['AvailabilityZones'])
curr_subnets_list = pcf_util.transform_list_of_dicts_to_desired_list(
curr_availabilty_zones, 'SubnetId')
des_subnets_list = self.desired_state_definition.get('Subnets')
if not pcf_util.is_list_equal(curr_subnets_list, des_subnets_list):
update = des_subnets_list
if update:
self.client.set_subnets(LoadBalancerArn=self.arn,
Subnets=list(update))
if self._need_update(curr_tags, des_tags):
add = list(
itertools.filterfalse(lambda x: x in curr_tags, des_tags))
remove = list(
itertools.filterfalse(lambda x: x in des_tags, curr_tags))
if remove:
self.client.remove_tags(ResourceArns=[self.arn],
TagKeys=[x.get('Key') for x in remove])
if add:
self.client.add_tags(ResourceArns=[self.arn], Tags=add)
if self._need_update(filtered_curr_listeners, des_listeners):
#Format current listener and desired listener to compare.
for item in des_listeners:
item.update({'Protocol': item.get('Protocol').upper()})
for item in curr_listeners:
self.client.delete_listener(ListenerArn=item['ListenerArn'])
for item in des_listeners:
item['LoadBalancerArn'] = self.arn
self.create_listener(item)
def _update(self):
"""
updates the ELB particle to match the desired state
"""
filtered_curr_def = {
key: self.current_state_definition[key]
for key in ElasticLoadBalancing.UPDATE_PARAM_FILTER
if key in self.current_state_definition.keys()
}
filtered_des_def = {
key: self.desired_state_definition[key]
for key in self.desired_state_definition.keys()
if key not in ElasticLoadBalancing.REMOVE_PARAM_FILTER
}
diff = pcf_util.diff_dict(filtered_curr_def, filtered_des_def)
curr_tags = self.client.describe_tags(LoadBalancerNames=[
self.elb_name,
])
curr_tags = curr_tags.get('TagDescriptions')[0].get('Tags')
curr_listeners = [
x['Listener']
for x in filtered_curr_def.get('ListenerDescriptions')
]
des_listeners = filtered_des_def['Listeners']
if diff.get('SecurityGroups', None):
self.client.apply_security_groups_to_load_balancer(
LoadBalancerName=self.elb_name,
SecurityGroups=filtered_des_def.get('SecurityGroups'))
if diff.get('Subnets', None):
remove = set(filtered_curr_def.get('Subnets')) - set(
filtered_des_def.get('Subnets'))
add = set(filtered_des_def.get('Subnets')) - set(
filtered_curr_def.get('Subnets'))
if remove:
self.client.detach_load_balancer_from_subnets(
LoadBalancerName=self.elb_name, Subnets=list(remove))
if add:
self.client.attach_load_balancer_to_subnets(
LoadBalancerName=self.elb_name, Subnets=list(add))
if _need_update(curr_tags, filtered_des_def.get('Tags', {})):
add = list(
itertools.filterfalse(lambda x: x in curr_tags,
filtered_des_def.get('Tags')))
remove = list(
itertools.filterfalse(
lambda x: x in filtered_des_def.get('Tags'), curr_tags))
if remove:
self.client.remove_tags(LoadBalancerNames=[self.elb_name],
Tags=[{
'Key': x.get('Key')
} for x in remove])
if add:
self.client.add_tags(LoadBalancerNames=[self.elb_name],
Tags=add)
if _need_update(curr_listeners, des_listeners):
#Format current listener and desired listener so that I can compare.
for item in des_listeners:
if not item.get('InstanceProtocol'):
item.update({'InstanceProtocol': item.get('Protocol')})
remove = list(
itertools.filterfalse(lambda x: x in des_listeners,
curr_listeners))
add = list(
itertools.filterfalse(lambda x: x in curr_listeners,
des_listeners))
if remove:
self.client.delete_load_balancer_listeners(
LoadBalancerName=self.elb_name,
LoadBalancerPorts=[x['LoadBalancerPort'] for x in remove])
if add:
self.client.create_load_balancer_listeners(
LoadBalancerName=self.elb_name, Listeners=add)
def test_update_dict_3():
orig_dict = {
"gg": "wp",
"lol": {
"cats": "grumpy"
},
"borg": ["resistance"],
"don't": "touch me"
}
updated_dict = {
"gg": "glhf",
"lol": {
"cats": "grumpy",
"rofl": "copter"
},
"borg": ["resistance", "is", "futile"]
}
diff_dict = pcf_util.diff_dict(orig_dict, updated_dict)
assert (orig_dict == {
"gg": "wp",
"lol": {
"cats": "grumpy"
},
"borg": ["resistance"],
"don't": "touch me"
})
assert (diff_dict == {
"borg": {
"original": ["resistance"],
"updated": ["resistance", "is", "futile"]
},
"lol": {
"rofl": {
"new": "copter"
}
},
"gg": {
"original": "wp",
"updated": "glhf"
}
})
new_dict, diff_dict = pcf_util.update_dict(orig_dict, updated_dict)
assert (new_dict == {
"gg": "glhf",
"lol": {
"cats": "grumpy",
"rofl": "copter"
},
"borg": ["resistance", "is", "futile"],
"don't": "touch me"
})
assert (diff_dict == {
"borg": {
"original": ["resistance"],
"updated": ["resistance", "is", "futile"]
},
"lol": {
"rofl": {
"new": "copter"
}
},
"gg": {
"original": "wp",
"updated": "glhf"
}
})
