def is_state_definition_equivalent(self): """ Checks if the current state definition and desired state definitions are equivalent including attributes. Returns: bool """ filtered_current_state_definition = pcf_util.param_filter( self.current_state_definition, BatchComputeEnvironment.UPDATE_PARAM_FILTER) filtered_desired_state_definition = pcf_util.param_filter( self.desired_state_definition, BatchComputeEnvironment.UPDATE_PARAM_FILTER) diff = pcf_util.diff_dict(filtered_desired_state_definition, filtered_current_state_definition) # TODO might need more comp logic here if not self.desired_state_definition[ "serviceRole"] in self.current_state_definition["serviceRole"]: return False if diff or len(diff) != 0: return False return True
def is_state_definition_equivalent(self): """ Determines if the current state definition matches the desired state definition. :return: bool """ self.sync_state() filtered_current_state_definition = pcf_util.param_filter( self.current_state_definition, RDS.UPDATE_PARAM_FILTER) filtered_desired_state_definition = pcf_util.param_filter( self.desired_state_definition, RDS.UPDATE_PARAM_FILTER) filtered_current_state_definition[ 'ApplyImmediately'] = self.desired_state_definition[ 'ApplyImmediately'] filtered_current_state_definition[ 'SkipFinalSnapshot'] = self.desired_state_definition[ 'SkipFinalSnapshot'] diff = pcf_util.diff_dict(filtered_current_state_definition, filtered_desired_state_definition) if not diff or len(diff) == 0: return True else: logger.debug( "State is not equivalent for {0} with diff: {1}".format( self.get_pcf_id(), json.dumps(diff))) return False
def is_state_definition_equivalent(self): """ Compared the desired state and current state definition Returns: bool """ self.sync_state() # use filters to remove any extra information self.current_state_definition = { key: self.current_state_definition[key] for key in SQSQueue.DEFINITION_FILTER if key in self.current_state_definition.keys() } self.desired_state_definition = { key: self.desired_state_definition[key] for key in SQSQueue.DEFINITION_FILTER if key in self.desired_state_definition.keys() } self.desired_state_definition["Attributes"] = { key: self.desired_state_definition.get("Attributes")[key] for key in SQSQueue.START_ATTR if key in self.desired_state_definition.get("Attributes").keys() } # only compare attributes specified in desired, ignore all else self.current_state_definition["Attributes"] = { key: self.current_state_definition.get("Attributes")[key] for key in self.desired_state_definition.get("Attributes").keys() if key in self.current_state_definition.get("Attributes").keys() } diff_dict = pcf_util.diff_dict(self.current_state_definition, self.desired_state_definition) return diff_dict == {}
def test_diff_dict(): orig = {'test': 72, 'test2': 66} update = { 'test': 72, } diff = pcf_util.diff_dict(orig, update) assert (diff == {})
def _update(self): # only updates InstanceCount desired_instances = self.desired_state_definition.get("Instances").get( "InstanceGroups") # instance pool if not desired_instances: desired_instances = self.desired_state_definition.get( "Instances").get("InstanceFleets") # TODO neither pool or group if not desired_instances: return True desired_instance_dict = pcf_util.list_to_dict("Name", desired_instances) desired_instance_dict = { k: pcf_util.param_filter(v, EMRCluster.FILTERED_UPDATE_PARAMS) for k, v in desired_instance_dict.items() } current_instance_dict = { k: pcf_util.keep_and_replace_keys(v, EMRCluster.UPDATE_PARAM_CONVERSIONS) for k, v in self.current_state_definition.get("Instances").items() } diff = pcf_util.diff_dict(current_instance_dict, desired_instance_dict) for k, v in diff.items(): curr_instance = self.current_state_definition["Instances"].get( k, {}).get('Id') #instance group if curr_instance and self.current_state_definition[ "Instances"].get(k, {}).get('InstanceGroupType'): self.client.modify_instance_groups( ClusterId=self._get_cluster_id(), InstanceGroups=[{ "InstanceGroupId": curr_instance, "InstanceCount": v["InstanceCount"]["updated"] }]) #instance pool elif curr_instance: od_cap = v.get("TargetOnDemandCapacity", {}).get("updated", 0) spot_cap = v.get("TargetSpotCapacity", {}).get("updated", 0) self.client.modify_instance_fleet( ClusterId=self._get_cluster_id(), InstanceFleet={ "InstanceFleetId": curr_instance, "TargetOnDemandCapacity": od_cap, "TargetSpotCapacity": spot_cap })
def is_state_definition_equivalent(self): """ Compare current and desired state definition Returns: bool """ self.sync_state() diff_dict = pcf_util.diff_dict(pcf_util.param_filter(self.current_state_definition, ECRRepository.DEFINITION_FILTER), pcf_util.param_filter(self.desired_state_definition, ECRRepository.DEFINITION_FILTER)) return diff_dict == {}
def is_state_definition_equivalent(self): """ Compared the desired state and current state definition Returns: bool """ self.current_state_definition = pcf_util.param_filter( self.current_state_definition, IAMRole.START_PARAMS_FILTER) self.current_state_definition['custom_config'] = {} self.get_iam_policies() diff_dict = {} if self.desired_state != State.terminated: #Comparing currently attached policies to desired policies try: attached_policy_arns = self.client.list_attached_role_policies( RoleName=self.role_name, PathPrefix=self.desired_state_definition.get('Path', '/')) except ClientError as e: #No attached policies attached_policy_arns = {} if attached_policy_arns.get('AttachedPolicies'): current_policy_arns = [ p.get('PolicyArn') for p in attached_policy_arns.get('AttachedPolicies') ] self.current_state_definition['custom_config'][ 'policy_arns'] = current_policy_arns else: self.current_state_definition['custom_config'][ 'policy_arns'] = [] if isinstance( self.desired_state_definition.get( 'AssumeRolePolicyDocument'), str): self.desired_state_definition[ 'AssumeRolePolicyDocument'] = json.loads( self.desired_state_definition.get( 'AssumeRolePolicyDocument')) self.current_state_definition['custom_config'][ 'IsInstanceProfile'] = self.custom_config.get( 'IsInstanceProfile', False) diff_dict = pcf_util.diff_dict(self.current_state_definition, self.desired_state_definition) return diff_dict == {}
def is_state_definition_equivalent(self): """ Compared the desired state and current state definition Returns: bool """ self.get_state() self.current_state_definition = pcf_util.param_filter( self.current_state_definition, CloudWatchEvent.START_PARAM_FILTER) desired_definition = pcf_util.param_filter( self.desired_state_definition, CloudWatchEvent.START_PARAM_FILTER) diff_dict = pcf_util.diff_dict(self.current_state_definition, desired_definition) return diff_dict == {}
def is_state_definition_equivalent(self): """ Compared the desired state and current state definition Returns: bool """ self.sync_state() # remove extra information self.current_state_definition = pcf_util.param_filter(self.current_state_definition, CloudWatchLog.START_PARAM_FILTER) # get the tags and add them on (they're together when creating, but separate when retrieving) self.current_state_definition["tags"] = self.client.list_tags_log_group(logGroupName= self.group_name).get("tags") self.desired_state_definition = pcf_util.param_filter(self.desired_state_definition, CloudWatchLog.START_PARAM_FILTER) diff_dict = pcf_util.diff_dict(self.current_state_definition, self.desired_state_definition) return diff_dict == {}
def is_state_definition_equivalent(self): """ Determines if the current state definition and the desired state definition are equivalent. Returns: bool """ self.sync_state() filtered_curr_def = { key: self.current_state_definition[key] for key in ElasticLoadBalancing.UPDATE_PARAM_FILTER if key in self.current_state_definition.keys() } filtered_des_def = { key: self.desired_state_definition[key] for key in self.desired_state_definition.keys() if key not in ElasticLoadBalancing.REMOVE_PARAM_FILTER } def_diff = pcf_util.diff_dict(filtered_curr_def, filtered_des_def) curr_tags = self.client.describe_tags(LoadBalancerNames=[ self.elb_name, ]) curr_tags = curr_tags.get('TagDescriptions')[0].get('Tags') des_tags = filtered_des_def.get('Tags') curr_listeners = [ x['Listener'] for x in filtered_curr_def.get('ListenerDescriptions') ] des_listeners = filtered_des_def['Listeners'] for item in des_listeners: item.update({'InstanceProtocol': item.get('Protocol').upper()}) item.update({'Protocol': item.get('Protocol').upper()}) def_diff.pop('Listeners', None) def_diff.pop('Tags', None) if def_diff or _need_update(curr_listeners, des_listeners) or _need_update( curr_tags, des_tags): return False return True
def is_state_definition_equivalent(self): """ Determines if the current state definition and the desired state definition are equivalent. Returns: bool """ filtered_curr_def = pcf_util.param_filter( self.current_state_definition, self.UPDATE_PARAM_FILTER) filtered_des_def = pcf_util.param_filter(self.desired_state_definition, self.REMOVE_PARAM_FILTER, True) def_diff = pcf_util.diff_dict(filtered_curr_def, filtered_des_def) curr_tags = self.client.describe_tags(ResourceArns=[self.arn]) curr_tags = curr_tags.get('TagDescriptions')[0].get('Tags', None) des_tags = self.desired_state_definition.get('Tags', None) curr_availabilty_zones = pcf_util.find_nested_dict_value( self.current_state_definition, ['AvailabilityZones']) curr_subnets_list = pcf_util.transform_list_of_dicts_to_desired_list( curr_availabilty_zones, 'SubnetId') des_subnets_list = self.desired_state_definition.get('Subnets') curr_listeners = [ pcf_util.param_filter(x, {'SslPolicy'}, True) for x in self.get_listener_status() ] des_listeners = self.custom_config['Listeners'] for item in des_listeners: item.update({'Protocol': item.get('Protocol').upper()}) for item in curr_listeners: if 'ListenerArn' in item: item.pop('ListenerArn') if def_diff or self._need_update(curr_tags, des_tags) or self._need_update(curr_listeners, des_listeners) \ or not pcf_util.is_list_equal(curr_subnets_list, des_subnets_list): return False return True
def is_state_definition_equivalent(self): """ Checks if the current state definition and desired state definitions are equivalent including attributes. Returns: bool """ filtered_desired_def = pcf_util.param_filter( self.desired_state_definition, self.UPDATE_PARAM_FILTER) filtered_current_def = pcf_util.param_filter( self.current_state_definition, self.UPDATE_PARAM_FILTER) diff = pcf_util.diff_dict(filtered_desired_def, filtered_current_def) if diff or len(diff) != 0: return False return True
def is_state_definition_equivalent(self): """ Compares the desired state and current state definitions. Returns: bool """ self.current_state_definition = pcf_util.param_filter( self.current_state_definition, CloudFormationStack.PARAM_FILTER) #seperate boto call to get the cloudformation template response = self.client.get_template(StackName=self.stack_name, TemplateStage="Original") self.current_state_definition['TemplateBody'] = json.dumps( response.get('TemplateBody')) diff_dict = pcf_util.diff_dict(self.current_state_definition, self.desired_state_definition) return diff_dict == {}
def is_state_definition_equivalent(self): """ Determines if the current state definition matches the current state definition. Uses keep and remove function from pcf util to remove extra params in desired state that are not in the current state. Returns: bool """ self.get_state() desired_definition = pcf_util.param_filter( self.desired_state_definition, AutoScalingGroup.REMOVE_PARAM_FILTER, True) diff = pcf_util.diff_dict(self.current_state_definition, desired_definition) if not diff or len(diff) == 0: return True else: logger.debug( "State is not equivalent for {0} with diff: {1}".format( self.get_pcf_id(), json.dumps(diff))) return False
def is_state_definition_equivalent(self): """ Compares the desired state and current state definitions. Returns: bool """ self.current_state_definition = pcf_util.param_filter( self.current_state_definition, CloudFormationStack.PARAM_FILTER) #seperate boto call to get the cloudformation template response = self.client.get_template(StackName=self.stack_name, TemplateStage="Original") # template comes back with extra quotes and extra escape \ for new lines. not sure why self.current_state_definition['TemplateBody'] = json.dumps( response.get('TemplateBody')).replace('\\n', '\n')[1:-1] filtered_desired_def = pcf_util.param_filter( self.desired_state_definition, CloudFormationStack.PARAM_FILTER) diff_dict = pcf_util.diff_dict(self.current_state_definition, filtered_desired_def) return diff_dict == {}
def is_state_definition_equivalent(self): """ Determines if state definitions are equivalent Returns: bool """ self.get_state() diff = pcf_util.diff_dict(self.current_state_definition, self.get_desired_state_definition()) if not diff or len(diff) == 0: return True else: # can't json dump function if diff.get('callbacks'): diff['callbacks'] = {'new': '<function>'} logger.debug( "State is not equivalent for {0} with diff: {1}".format( self.get_pcf_id(), json.dumps(diff))) return False
def is_state_definition_equivalent(self): """ Determines if current state is equivalent to the desired state. Returns: bool """ desired_instances = self.desired_state_definition.get("Instances").get( "InstanceGroups") # instance pool if not desired_instances: desired_instances = self.desired_state_definition.get( "Instances").get("InstanceFleets") # TODO neither pool or group if not desired_instances: return True desired_instance_dict = pcf_util.list_to_dict("Name", desired_instances) desired_instance_dict = { k: pcf_util.param_filter(v, EMRCluster.FILTERED_UPDATE_PARAMS) for k, v in desired_instance_dict.items() } current_instance_dict = { k: pcf_util.keep_and_replace_keys(v, EMRCluster.UPDATE_PARAM_CONVERSIONS) for k, v in self.current_state_definition.get("Instances").items() } # moto bug with instance fleets if not current_instance_dict: return True diff = pcf_util.diff_dict(current_instance_dict, desired_instance_dict) return diff == {}
def is_state_definition_equivalent(self): """ Determines if the current state definition matches the current state definition. Uses keep and remove function from pcf util to remove extra params in desired state that are not in the current state. :return: bool """ self.get_state() diff = pcf_util.diff_dict( self.current_state_definition, pcf_util.keep_and_remove_keys( self.get_desired_state_definition(), Route53Record.REMOVE_PARAM_CONVERSIONS)) if not diff or len(diff) == 0: return True elif self.state == State.terminated: return True else: logger.debug( "State is not equivalent for {0} with diff: {1}".format( self.get_pcf_id(), json.dumps(diff))) return False
def is_state_definition_equivalent(self): """ Determines if state definitions are equivalent including list of tags. Returns: Bool """ if len(self.current_state_definition.get('Tags', [])) != len( self.tags): return False tag_zip = zip( sorted(self.current_state_definition.get('Tags', []), key=lambda k: k["Value"]), sorted(self.tags, key=lambda k: k["Value"])) diff_tags = any(x != y for x, y in tag_zip) if diff_tags: return False filtered_current_definition = pcf_util.param_filter( self.get_current_state_definition(), EBSVolume.UPDATE_PARAM_FILTER, ) filtered_desired_definition = pcf_util.param_filter( self.get_desired_state_definition(), EBSVolume.UPDATE_PARAM_FILTER, ) diff = pcf_util.diff_dict(filtered_current_definition, filtered_desired_definition) if diff: return False return True
def _update(self): """ Calls boto3 modify_volume(). If tags are changes then also calls delete_tags() and then create_tags() Returns: boto3 modify_volume() response """ filtered_current_definition = pcf_util.param_filter( self.get_current_state_definition(), EBSVolume.UPDATE_PARAM_FILTER, ) filtered_desired_definition = pcf_util.param_filter( self.get_desired_state_definition(), EBSVolume.UPDATE_PARAM_FILTER, ) diff = pcf_util.diff_dict(filtered_current_definition, filtered_desired_definition) # update tags tag_zip = zip( sorted(self.current_state_definition.get('Tags', []), key=lambda k: k["Value"]), sorted(self.tags, key=lambda k: k["Value"])) diff_tags = any(x != y for x, y in tag_zip) if diff_tags or len(self.current_state_definition.get( 'Tags', [])) != len(self.tags): self.client.delete_tags(Resources=[self.volume_id]) self.client.create_tags(Resources=[self.volume_id], Tags=self.tags) if not diff: return return self.client.modify_volume(VolumeId=self.volume_id, **filtered_desired_definition)
def _update(self): """ updates the alb particle to match the desired state """ filtered_curr_def = pcf_util.param_filter( self.current_state_definition, self.UPDATE_PARAM_FILTER) filtered_des_def = pcf_util.param_filter(self.desired_state_definition, self.REMOVE_PARAM_FILTER, True) diff = pcf_util.diff_dict(filtered_curr_def, filtered_des_def) curr_tags = self.client.describe_tags(ResourceArns=[self.arn]) curr_tags = curr_tags.get('TagDescriptions')[0].get('Tags') des_tags = self.desired_state_definition.get('Tags', [{}]) curr_listeners = [x for x in self.get_listener_status()] # Format current listener to compare. filtered_curr_listeners = [x for x in self.get_listener_status()] for item in filtered_curr_listeners: if 'ListenerArn' in item: item.pop('ListenerArn') des_listeners = self.custom_config['Listeners'] if diff.get('SecurityGroups', None): self.client.set_security_groups( LoadBalancerArn=self.arn, SecurityGroups=filtered_des_def.get('SecurityGroups')) curr_availabilty_zones = pcf_util.find_nested_dict_value( self.current_state_definition, ['AvailabilityZones']) curr_subnets_list = pcf_util.transform_list_of_dicts_to_desired_list( curr_availabilty_zones, 'SubnetId') des_subnets_list = self.desired_state_definition.get('Subnets') if not pcf_util.is_list_equal(curr_subnets_list, des_subnets_list): update = des_subnets_list if update: self.client.set_subnets(LoadBalancerArn=self.arn, Subnets=list(update)) if self._need_update(curr_tags, des_tags): add = list( itertools.filterfalse(lambda x: x in curr_tags, des_tags)) remove = list( itertools.filterfalse(lambda x: x in des_tags, curr_tags)) if remove: self.client.remove_tags(ResourceArns=[self.arn], TagKeys=[x.get('Key') for x in remove]) if add: self.client.add_tags(ResourceArns=[self.arn], Tags=add) if self._need_update(filtered_curr_listeners, des_listeners): #Format current listener and desired listener to compare. for item in des_listeners: item.update({'Protocol': item.get('Protocol').upper()}) for item in curr_listeners: self.client.delete_listener(ListenerArn=item['ListenerArn']) for item in des_listeners: item['LoadBalancerArn'] = self.arn self.create_listener(item)
def _update(self): """ updates the ELB particle to match the desired state """ filtered_curr_def = { key: self.current_state_definition[key] for key in ElasticLoadBalancing.UPDATE_PARAM_FILTER if key in self.current_state_definition.keys() } filtered_des_def = { key: self.desired_state_definition[key] for key in self.desired_state_definition.keys() if key not in ElasticLoadBalancing.REMOVE_PARAM_FILTER } diff = pcf_util.diff_dict(filtered_curr_def, filtered_des_def) curr_tags = self.client.describe_tags(LoadBalancerNames=[ self.elb_name, ]) curr_tags = curr_tags.get('TagDescriptions')[0].get('Tags') curr_listeners = [ x['Listener'] for x in filtered_curr_def.get('ListenerDescriptions') ] des_listeners = filtered_des_def['Listeners'] if diff.get('SecurityGroups', None): self.client.apply_security_groups_to_load_balancer( LoadBalancerName=self.elb_name, SecurityGroups=filtered_des_def.get('SecurityGroups')) if diff.get('Subnets', None): remove = set(filtered_curr_def.get('Subnets')) - set( filtered_des_def.get('Subnets')) add = set(filtered_des_def.get('Subnets')) - set( filtered_curr_def.get('Subnets')) if remove: self.client.detach_load_balancer_from_subnets( LoadBalancerName=self.elb_name, Subnets=list(remove)) if add: self.client.attach_load_balancer_to_subnets( LoadBalancerName=self.elb_name, Subnets=list(add)) if _need_update(curr_tags, filtered_des_def.get('Tags', {})): add = list( itertools.filterfalse(lambda x: x in curr_tags, filtered_des_def.get('Tags'))) remove = list( itertools.filterfalse( lambda x: x in filtered_des_def.get('Tags'), curr_tags)) if remove: self.client.remove_tags(LoadBalancerNames=[self.elb_name], Tags=[{ 'Key': x.get('Key') } for x in remove]) if add: self.client.add_tags(LoadBalancerNames=[self.elb_name], Tags=add) if _need_update(curr_listeners, des_listeners): #Format current listener and desired listener so that I can compare. for item in des_listeners: if not item.get('InstanceProtocol'): item.update({'InstanceProtocol': item.get('Protocol')}) remove = list( itertools.filterfalse(lambda x: x in des_listeners, curr_listeners)) add = list( itertools.filterfalse(lambda x: x in curr_listeners, des_listeners)) if remove: self.client.delete_load_balancer_listeners( LoadBalancerName=self.elb_name, LoadBalancerPorts=[x['LoadBalancerPort'] for x in remove]) if add: self.client.create_load_balancer_listeners( LoadBalancerName=self.elb_name, Listeners=add)
def test_update_dict_3(): orig_dict = { "gg": "wp", "lol": { "cats": "grumpy" }, "borg": ["resistance"], "don't": "touch me" } updated_dict = { "gg": "glhf", "lol": { "cats": "grumpy", "rofl": "copter" }, "borg": ["resistance", "is", "futile"] } diff_dict = pcf_util.diff_dict(orig_dict, updated_dict) assert (orig_dict == { "gg": "wp", "lol": { "cats": "grumpy" }, "borg": ["resistance"], "don't": "touch me" }) assert (diff_dict == { "borg": { "original": ["resistance"], "updated": ["resistance", "is", "futile"] }, "lol": { "rofl": { "new": "copter" } }, "gg": { "original": "wp", "updated": "glhf" } }) new_dict, diff_dict = pcf_util.update_dict(orig_dict, updated_dict) assert (new_dict == { "gg": "glhf", "lol": { "cats": "grumpy", "rofl": "copter" }, "borg": ["resistance", "is", "futile"], "don't": "touch me" }) assert (diff_dict == { "borg": { "original": ["resistance"], "updated": ["resistance", "is", "futile"] }, "lol": { "rofl": { "new": "copter" } }, "gg": { "original": "wp", "updated": "glhf" } })