def is_state_definition_equivalent(self):
"""
Checks if the current state definition and desired state definitions are equivalent including
attributes.
Returns:
bool
"""
filtered_current_state_definition = pcf_util.param_filter(
self.current_state_definition,
BatchComputeEnvironment.UPDATE_PARAM_FILTER)
filtered_desired_state_definition = pcf_util.param_filter(
self.desired_state_definition,
BatchComputeEnvironment.UPDATE_PARAM_FILTER)
diff = pcf_util.diff_dict(filtered_desired_state_definition,
filtered_current_state_definition)
# TODO might need more comp logic here
if not self.desired_state_definition[
"serviceRole"] in self.current_state_definition["serviceRole"]:
return False
if diff or len(diff) != 0:
return False
return True
def is_state_definition_equivalent(self):
"""
Determines if the current state definition matches the desired state definition.
:return: bool
"""
self.sync_state()
filtered_current_state_definition = pcf_util.param_filter(
self.current_state_definition, RDS.UPDATE_PARAM_FILTER)
filtered_desired_state_definition = pcf_util.param_filter(
self.desired_state_definition, RDS.UPDATE_PARAM_FILTER)
filtered_current_state_definition[
'ApplyImmediately'] = self.desired_state_definition[
'ApplyImmediately']
filtered_current_state_definition[
'SkipFinalSnapshot'] = self.desired_state_definition[
'SkipFinalSnapshot']
diff = pcf_util.diff_dict(filtered_current_state_definition,
filtered_desired_state_definition)
if not diff or len(diff) == 0:
return True
else:
logger.debug(
"State is not equivalent for {0} with diff: {1}".format(
self.get_pcf_id(), json.dumps(diff)))
return False
def is_state_definition_equivalent(self):
"""
Compares the desired state and current state definitions.
Returns:
bool
"""
self.get_state()
current_definition = pcf_util.param_filter(
self.current_state_definition, DynamoDB.REMOVE_PARAM_FILTER, True)
desired_definition = pcf_util.param_filter(
self.desired_state_definition, DynamoDB.START_PARAM_FILTER)
# compare tags
if self._arn:
current_tags = self.client.list_tags_of_resource(
ResourceArn=self._arn)["Tags"]
else:
current_tags = self.client.list_tags_of_resource(
ResourceArn=self.current_state_definition.get(
"TableArn"))["Tags"]
desired_tags = desired_definition.get("Tags", [])
new_desired_state_def, diff_dict = pcf_util.update_dict(
current_definition, desired_definition)
# self.create_table() does not return "Tags" as an attribute therefore, current_state_definition does not have
# any reference to Tags, so it is removed from the comparison between current_definition and desired_definition
diff_dict.pop('Tags', None)
return diff_dict == {} and not self._need_update(
current_tags, desired_tags)
def is_state_definition_equivalent(self):
"""
Compare current and desired state definition
Returns:
bool
"""
self.sync_state()
diff_dict = pcf_util.diff_dict(pcf_util.param_filter(self.current_state_definition, ECRRepository.DEFINITION_FILTER),
pcf_util.param_filter(self.desired_state_definition, ECRRepository.DEFINITION_FILTER))
return diff_dict == {}
def _update(self):
"""
Updates the dynamodb particle to match current state definition.
"""
new_desired_state_def, diff_dict = pcf_util.update_dict(
self.current_state_definition, self.get_desired_state_definition())
new_desired_state_def["ProvisionedThroughput"] = pcf_util.param_filter(
new_desired_state_def["ProvisionedThroughput"],
DynamoDB.THROUGHPUT_PARAM_FILTER)
update_definition = pcf_util.param_filter(new_desired_state_def,
DynamoDB.UPDATE_PARAM_FILTER)
self.client.update_table(**update_definition)
def is_state_definition_equivalent(self):
"""
Compares the desired state and current state definitions.
Returns:
bool
"""
self.get_state()
current_definition = pcf_util.param_filter(
self.current_state_definition, DynamoDB.REMOVE_PARAM_FILTER, True)
desired_definition = pcf_util.param_filter(
self.desired_state_definition, DynamoDB.START_PARAM_FILTER)
new_desired_state_def, diff_dict = pcf_util.update_dict(
current_definition, desired_definition)
return diff_dict == {}
def is_state_definition_equivalent(self):
"""
Compared the desired state and current state definition
Returns:
bool
"""
self.get_state()
self.current_state_definition = pcf_util.param_filter(
self.current_state_definition, CloudWatchEvent.START_PARAM_FILTER)
desired_definition = pcf_util.param_filter(
self.desired_state_definition, CloudWatchEvent.START_PARAM_FILTER)
diff_dict = pcf_util.diff_dict(self.current_state_definition,
desired_definition)
return diff_dict == {}
def _start(self):
"""
Required method: Used in methods in the base class(es).
Creates a customer-managed key (CMK), or if the alias specified already exists, re-enables
an existing one. The functionality needs to be mixed, since this function is called in both
the Terminated --> Running and the Stopped --> Running state transitions. Return object is
supplied for convenience, and is not used in any operations.
Returns:
dict (response of boto3 kms create_key or enable_key, depending on transition type)
"""
if self.current_state_definition:
return self.client.enable_key(
KeyId=self.current_state_definition.get('KeyId'))
else:
# TODO add check for KeyId in desired state definition so existing keys can be reused
start_definition = pcf_util.param_filter(
self.get_desired_state_definition(), KMSKey.START_PARAMS)
create_response = self.client.create_key(**start_definition)
# TODO add a check for uniqueness of alias
self.client.create_alias(
AliasName='alias/' + self.key_name,
TargetKeyId=create_response.get('KeyMetadata').get('KeyId'))
return create_response
def _start(self):
"""
Creates the S3 bucket
Adds Tags to the S3 bucket if specified in custom_config
Returns:
response of boto3 create_bucket
"""
create_definition = pcf_util.param_filter(self.get_desired_state_definition(), S3Bucket.START_PARAMS_FILTER)
response = self.client.create_bucket(**create_definition)
if self.custom_config.get("Tags"):
tags = self.custom_config.get("Tags")
tag_set = []
for k, v in tags.items():
tag_set.append({
"Key": k,
"Value": v
})
self.client.put_bucket_tagging(
Bucket=self.bucket_name,
Tagging={
"TagSet":tag_set
}
)
return response
def _update(self):
"""
Calls boto3 put_attributes() to update ECS Instance attributes. Does not allow for attributes that start
with com.amazonaws.ecs. or instance-id to be updated.
Returns:
boto3 put_attributes() response
"""
new_desired_state_def, diff_dict = pcf_util.update_dict(
self.current_state_definition, self.get_desired_state_definition())
update_definition = pcf_util.param_filter(
new_desired_state_def, ECSInstance.UPDATE_PARAM_FILTER)
attributes = []
for a in update_definition['attributes']:
if (not a['name'].startswith('ecs.')
and not a['name'].startswith('com.amazonaws.ecs.')
and a['name'] not in ECSInstance.PROTECTED_ATTRIBUTES):
attributes.append({
'name': a['name'],
'value': a['value'],
'targetType': 'container-instance',
'targetId': self.get_ecs_instance_id(),
})
return self.client.put_attributes(
cluster=self.get_cluster_name(),
attributes=attributes,
)
def _start(self):
"""
Creates the IAM Role
Returns:
response of boto3 create_role
"""
create_definition = pcf_util.param_filter(
self.get_desired_state_definition(), IAMRole.START_PARAMS_FILTER)
try:
self.client.create_role(**create_definition)
except ClientError as e:
raise e
if self.custom_config.get('IsInstanceProfile', False):
try:
self.client.create_instance_profile(
InstanceProfileName=self.role_name)
except ClientError as e:
logger.info(e)
try:
self.client.add_role_to_instance_profile(
InstanceProfileName=self.role_name,
RoleName=self.role_name)
except ClientError as e:
logger.info(e)
def is_state_definition_equivalent(self):
"""
Compared the desired state and current state definition
Returns:
bool
"""
self.sync_state()
# remove extra information
self.current_state_definition = pcf_util.param_filter(self.current_state_definition, CloudWatchLog.START_PARAM_FILTER)
# get the tags and add them on (they're together when creating, but separate when retrieving)
self.current_state_definition["tags"] = self.client.list_tags_log_group(logGroupName=
self.group_name).get("tags")
self.desired_state_definition = pcf_util.param_filter(self.desired_state_definition, CloudWatchLog.START_PARAM_FILTER)
diff_dict = pcf_util.diff_dict(self.current_state_definition, self.desired_state_definition)
return diff_dict == {}
def is_state_definition_equivalent(self):
"""
Determines if the current state definition and the desired state definition are equivalent.
Returns:
bool
"""
filtered_curr_def = pcf_util.param_filter(
self.current_state_definition, self.UPDATE_PARAM_FILTER)
filtered_des_def = pcf_util.param_filter(self.desired_state_definition,
self.REMOVE_PARAM_FILTER,
True)
def_diff = pcf_util.diff_dict(filtered_curr_def, filtered_des_def)
curr_tags = self.client.describe_tags(ResourceArns=[self.arn])
curr_tags = curr_tags.get('TagDescriptions')[0].get('Tags', None)
des_tags = self.desired_state_definition.get('Tags', None)
curr_availabilty_zones = pcf_util.find_nested_dict_value(
self.current_state_definition, ['AvailabilityZones'])
curr_subnets_list = pcf_util.transform_list_of_dicts_to_desired_list(
curr_availabilty_zones, 'SubnetId')
des_subnets_list = self.desired_state_definition.get('Subnets')
curr_listeners = [
pcf_util.param_filter(x, {'SslPolicy'}, True)
for x in self.get_listener_status()
]
des_listeners = self.custom_config['Listeners']
for item in des_listeners:
item.update({'Protocol': item.get('Protocol').upper()})
for item in curr_listeners:
if 'ListenerArn' in item:
item.pop('ListenerArn')
if def_diff or self._need_update(curr_tags, des_tags) or self._need_update(curr_listeners, des_listeners) \
or not pcf_util.is_list_equal(curr_subnets_list, des_subnets_list):
return False
return True
def create(self):
"""
Creates an Application Load Balancer that matches current state definition
Returns:
response of boto3 create alb instance with action create_load_balancer.
"""
create_def = pcf_util.param_filter(self.desired_state_definition, self.START_PARAM_FILTER)
return self.client.create_load_balancer(**create_def)
def _update(self):
"""
Updates the rds instance that matches current state definition
:return: response of boto3 change rds instance with modify_db_instance.
"""
update_def = pcf_util.param_filter(self.desired_state_definition,
RDS.UPDATE_PARAM_FILTER)
return self.client.modify_db_instance(**update_def)
def _start(self):
"""
Starts the Cloudwatch Log Group according to the desired definition
Returns:
response of boto3 create_log_group
"""
start_definition = pcf_util.param_filter(self.desired_state_definition, CloudWatchLog.START_PARAM_FILTER)
return self.client.create_log_group(**start_definition)
def create(self):
"""
Creates the rds instance that matches current state definition
:return: response of boto3 create rds instance with action create_db_instance.
"""
create_def = pcf_util.param_filter(self.desired_state_definition,
RDS.START_PARAM_FILTER)
return self.client.create_db_instance(**create_def)
def _start(self):
"""
Starts the events particle that matches desired state definition
Returns:
response of boto3 create_repository()
"""
start_definition = pcf_util.param_filter(self.get_desired_state_definition(), ECRRepository.DEFINITION_FILTER)
return self.client.create_repository(**start_definition)
def _start(self):
"""
Starts the dynamodb particle that matches the desired state function
Returns:
reponse of boto3 create_table
"""
start_definition = pcf_util.param_filter(
self.get_desired_state_definition(), DynamoDB.START_PARAM_FILTER)
return self.client.create_table(**start_definition)
def is_state_definition_equivalent(self):
"""
Compares the desired state and current state definition and returns whether they are equivalent
Only considers fields defined in the desired definition
All fields not specified are left alone in the current state (excluding rules)
Both rules lists must be defined even when empty
Returns:
bool
"""
self.sync_state()
# use filters to remove any extra information
desired_config = pcf_util.param_filter(
self.desired_state_definition.get("custom_config", {}),
SecurityGroup.DEFINITION_FILTER)
current_config = pcf_util.param_filter(
self.get_current_state_definition(), desired_config.keys())
diff = DeepDiff(current_config, desired_config, ignore_order=True)
return diff == {}
def _start(self):
"""
start the autoscaling group particle that matches current state definition
Returns:
response of boto3 create_auto_scaling_group()
"""
start_definition = pcf_util.param_filter(
self.desired_state_definition, AutoScalingGroup.START_PARAM_FILTER)
return self.client.create_auto_scaling_group(**start_definition)
def _start(self):
"""
Starts the SQS Queue according to the desired definition
Returns:
response of boto3 create_queue
"""
start_definition = pcf_util.param_filter(
self.get_desired_state_definition(), SQSQueue.START_PARAM_FILTER)
if "Attributes" in start_definition.keys():
start_definition["Attributes"] = pcf_util.param_filter(
start_definition["Attributes"], SQSQueue.START_ATTR)
response = self.client.create_queue(**start_definition)
self.queue_url = response["QueueUrl"]
if self.desired_state_definition.get("Tags"):
self.client.tag_queue(
QueueUrl=self.queue_url,
Tags=self.desired_state_definition.get("Tags"))
return response
def is_state_definition_equivalent(self):
"""
Checks if the current state definition and desired state definitions are equivalent including
attributes.
Returns:
bool
"""
filtered_desired_def = pcf_util.param_filter(
self.desired_state_definition, self.UPDATE_PARAM_FILTER)
filtered_current_def = pcf_util.param_filter(
self.current_state_definition, self.UPDATE_PARAM_FILTER)
diff = pcf_util.diff_dict(filtered_desired_def, filtered_current_def)
if diff or len(diff) != 0:
return False
return True
def is_state_definition_equivalent(self):
"""
Compares the desired state and current state definitions.
Returns:
bool
"""
self.get_state()
desired_definition = pcf_util.param_filter(self.desired_state_definition, LambdaFunction.RETURN_PARAM_FILTER)
new_desired_state_def, diff_dict = pcf_util.update_dict(self.current_state_definition, desired_definition)
return diff_dict == {}
def create(self):
"""
Calls boto3 create_instances(). This is called for terminated to running state transition.
Returns:
boto3 create_instances() response
"""
create_definition = pcf_util.param_filter(
self.get_desired_state_definition(),
EC2Instance.START_PARAM_FILTER)
return self.resource.create_instances(**create_definition)
def _update(self):
"""
Updates the Auto Scaling group that matches current state definition
Returns:
response of boto3 update_auto_scaling_group()
"""
update_definition = pcf_util.param_filter(
self.desired_state_definition,
AutoScalingGroup.REMOVE_PARAM_FILTER, True)
return self.client.update_auto_scaling_group(**update_definition)
def _update(self):
"""
Calls boto3 update_job_queue(). Can only update fields in UPDATE_PARAM_FILTER.
Returns:
boto3 put_attributes() response
"""
filtered_definition = pcf_util.param_filter(
self.desired_state_definition, self.UPDATE_PARAM_FILTER)
return self.client.update_job_queue(jobQueue=self.name,
**filtered_definition)
def _start(self):
"""
Starts the events particle that matches desired state definition
Returns:
response of boto3 put_rule
"""
start_definition = pcf_util.param_filter(
self.get_desired_state_definition(),
CloudWatchEvent.START_PARAM_FILTER)
return self.client.put_rule(**start_definition)
def _update(self):
# only updates InstanceCount
desired_instances = self.desired_state_definition.get("Instances").get(
"InstanceGroups")
# instance pool
if not desired_instances:
desired_instances = self.desired_state_definition.get(
"Instances").get("InstanceFleets")
# TODO neither pool or group
if not desired_instances:
return True
desired_instance_dict = pcf_util.list_to_dict("Name",
desired_instances)
desired_instance_dict = {
k: pcf_util.param_filter(v, EMRCluster.FILTERED_UPDATE_PARAMS)
for k, v in desired_instance_dict.items()
}
current_instance_dict = {
k:
pcf_util.keep_and_replace_keys(v,
EMRCluster.UPDATE_PARAM_CONVERSIONS)
for k, v in self.current_state_definition.get("Instances").items()
}
diff = pcf_util.diff_dict(current_instance_dict, desired_instance_dict)
for k, v in diff.items():
curr_instance = self.current_state_definition["Instances"].get(
k, {}).get('Id')
#instance group
if curr_instance and self.current_state_definition[
"Instances"].get(k, {}).get('InstanceGroupType'):
self.client.modify_instance_groups(
ClusterId=self._get_cluster_id(),
InstanceGroups=[{
"InstanceGroupId":
curr_instance,
"InstanceCount":
v["InstanceCount"]["updated"]
}])
#instance pool
elif curr_instance:
od_cap = v.get("TargetOnDemandCapacity", {}).get("updated", 0)
spot_cap = v.get("TargetSpotCapacity", {}).get("updated", 0)
self.client.modify_instance_fleet(
ClusterId=self._get_cluster_id(),
InstanceFleet={
"InstanceFleetId": curr_instance,
"TargetOnDemandCapacity": od_cap,
"TargetSpotCapacity": spot_cap
})
def _start(self):
"""
Starts the lambda particle that matches desired state definition
:return: response of boto3 create_function
"""
new_desired_state_def, diff_dict = pcf_util.update_dict(self.current_state_definition, self.get_desired_state_definition())
start_definition = pcf_util.param_filter(new_desired_state_def, LambdaFunction.START_PARAM_FILTER)
if self.is_zip_local:
start_definition["Code"]["ZipFile"] = self.file_get_contents()
self.client.create_function(**start_definition)
