def test_update_dict(): orig_dict = {} updated_dict = { "gg": "wp", "lol": { "cats": "grumpy" }, "borg": ["resistance", "is", "futile"] } new_dict, diff_dict = pcf_util.update_dict(orig_dict, updated_dict) assert (orig_dict == {}) assert (new_dict == updated_dict) assert (diff_dict == { "gg": { "new": "wp" }, "lol": { "new": { "cats": "grumpy" } }, "borg": { "new": ["resistance", "is", "futile"] } })
def _update(self): """ Calls boto3 put_attributes() to update ECS Instance attributes. Does not allow for attributes that start with com.amazonaws.ecs. or instance-id to be updated. Returns: boto3 put_attributes() response """ new_desired_state_def, diff_dict = pcf_util.update_dict( self.current_state_definition, self.get_desired_state_definition()) update_definition = pcf_util.param_filter( new_desired_state_def, ECSInstance.UPDATE_PARAM_FILTER) attributes = [] for a in update_definition['attributes']: if (not a['name'].startswith('ecs.') and not a['name'].startswith('com.amazonaws.ecs.') and a['name'] not in ECSInstance.PROTECTED_ATTRIBUTES): attributes.append({ 'name': a['name'], 'value': a['value'], 'targetType': 'container-instance', 'targetId': self.get_ecs_instance_id(), }) return self.client.put_attributes( cluster=self.get_cluster_name(), attributes=attributes, )
def is_state_definition_equivalent(self): """ Compares the desired state and current state definitions. Returns: bool """ self.get_state() current_definition = pcf_util.param_filter( self.current_state_definition, DynamoDB.REMOVE_PARAM_FILTER, True) desired_definition = pcf_util.param_filter( self.desired_state_definition, DynamoDB.START_PARAM_FILTER) # compare tags if self._arn: current_tags = self.client.list_tags_of_resource( ResourceArn=self._arn)["Tags"] else: current_tags = self.client.list_tags_of_resource( ResourceArn=self.current_state_definition.get( "TableArn"))["Tags"] desired_tags = desired_definition.get("Tags", []) new_desired_state_def, diff_dict = pcf_util.update_dict( current_definition, desired_definition) # self.create_table() does not return "Tags" as an attribute therefore, current_state_definition does not have # any reference to Tags, so it is removed from the comparison between current_definition and desired_definition diff_dict.pop('Tags', None) return diff_dict == {} and not self._need_update( current_tags, desired_tags)
def test_troll_update_string(): orig_dict = {"lolcats": "lol"} updated_dict = {"lolcats": ""} new_dict, diff_dict = pcf_util.update_dict(orig_dict, updated_dict) assert (new_dict == updated_dict) assert (diff_dict == { 'lolcats': { 'original': "lol", 'updated': "", } })
def test_troll_update_list(): orig_dict = {"lolcats": ["fly"]} updated_dict = {"lolcats": []} new_dict, diff_dict = pcf_util.update_dict(orig_dict, updated_dict) assert (new_dict == updated_dict) assert (diff_dict == { 'lolcats': { 'original': ["fly"], 'updated': [], } })
def is_state_definition_equivalent(self): """ Compares the desired state and current state definitions. Returns: bool """ self.get_state() desired_definition = pcf_util.param_filter(self.desired_state_definition, LambdaFunction.RETURN_PARAM_FILTER) new_desired_state_def, diff_dict = pcf_util.update_dict(self.current_state_definition, desired_definition) return diff_dict == {}
def _start(self): """ Starts the lambda particle that matches desired state definition :return: response of boto3 create_function """ new_desired_state_def, diff_dict = pcf_util.update_dict(self.current_state_definition, self.get_desired_state_definition()) start_definition = pcf_util.param_filter(new_desired_state_def, LambdaFunction.START_PARAM_FILTER) if self.is_zip_local: start_definition["Code"]["ZipFile"] = self.file_get_contents() self.client.create_function(**start_definition)
def _update(self): """ Updates the dynamodb particle to match current state definition. """ new_desired_state_def, diff_dict = pcf_util.update_dict( self.current_state_definition, self.get_desired_state_definition()) new_desired_state_def["ProvisionedThroughput"] = pcf_util.param_filter( new_desired_state_def["ProvisionedThroughput"], DynamoDB.THROUGHPUT_PARAM_FILTER) update_definition = pcf_util.param_filter(new_desired_state_def, DynamoDB.UPDATE_PARAM_FILTER) self.client.update_table(**update_definition)
def _start(self): """ Calls boto3 register_task_definition() Returns: boto3 register_task_definition() response """ new_desired_state_def, diff_dict = pcf_util.update_dict(self.current_state_definition, self.get_desired_state_definition()) new_desired_state_def = pcf_util.param_filter(new_desired_state_def, ECSTaskDefinition.START_PARAM_FILTER) resp = self.client.register_task_definition(**new_desired_state_def) self.current_state_definition = resp return resp
def test_troll_update_dict_2(): orig_dict = {"lolcats": {"eat": "freefood"}} updated_dict = {"lolcats": None} new_dict, diff_dict = pcf_util.update_dict(orig_dict, updated_dict) assert (new_dict == updated_dict) assert (diff_dict == { 'lolcats': { 'original': { "eat": "freefood" }, 'updated': None, } })
def update_particle_definition(self, particle_definition, base_particle): """ Updates a particle's definition with a previous definition. Replaces any fields that are specified. Args: particle_definition (dict): new particle defintion (can be empty) base_particle (particle): the particle that the new particle is inheriting from Returns: updated_particle_definition """ updated_particle_defintion, diff_dict = pcf_util.update_dict(base_particle.particle_definition, particle_definition) return updated_particle_defintion
def _update(self): """ Calls boto3 update_service() Returns: boto3 update_service() response """ new_desired_state_def, diff_dict = pcf_util.update_dict( self.current_state_definition, self.get_desired_state_definition()) update_definition = pcf_util.keep_and_replace_keys( new_desired_state_def, ECSService.UPDATE_PARAM_CONVERSIONS) return self.client.update_service(**update_definition)
def is_state_definition_equivalent(self): """ Compares the desired state and current state definitions. Returns: bool """ self.get_state() current_definition = pcf_util.param_filter( self.current_state_definition, DynamoDB.REMOVE_PARAM_FILTER, True) desired_definition = pcf_util.param_filter( self.desired_state_definition, DynamoDB.START_PARAM_FILTER) new_desired_state_def, diff_dict = pcf_util.update_dict( current_definition, desired_definition) return diff_dict == {}
def _start(self): """ Calls boto3 run_task function to create a new task. If successful this gets the arn of the task and adds it to the current_state_definition. """ new_desired_state_def, diff_dict = pcf_util.update_dict( self.current_state_definition, self.get_desired_state_definition()) start_definition = pcf_util.keep_and_replace_keys( new_desired_state_def, ECSTask.START_PARAM_CONVERSIONS) self.sync_state() if self.state == State.stopped and "containers" in self.current_state_definition: containers = self.current_state_definition.get("containers") for container in containers: if container.get("exitCode", -1) != 0: logger.warning( "Task {} failed to execute with exit code: {} and reason: {}... setting desired state to {}" .format(self.get_task_arn(), container.get("exitCode"), container.get("reason"), State.stopped)) self.failure_reason = { "type": "container", "reason": self.current_state_definition.get( "stoppedReason", "N/A") } self.set_desired_state(State.stopped) return resp = self.client.run_task(**start_definition) task = resp.get("tasks", []) failures = resp.get("failures", []) if len(task) == 1: self.current_state_definition["taskArn"] = task[0].get("taskArn") elif len(task) == 0 and len(failures) > 0: logger.warning( "Task {} failed to be placed due to an ECS error: {}... setting desired state to {}" .format(self.get_task_arn(), failures[0].get("reason"), State.stopped)) self.failure_reason = { "type": "ecs", "reason": failures[0].get("reason") } self.set_desired_state(State.stopped) else: Exception("ECS Task failed to start")
def _start(self): """ Calls boto3 create_volume(). Returns: boto3 create_volume() response """ new_desired_state_def, diff_dict = pcf_util.update_dict( self.current_state_definition, self.get_desired_state_definition()) start_params = pcf_util.param_filter(new_desired_state_def, EBSVolume.START_PARAM_FILTER) res = self.client.create_volume(**start_params) if res.get('VolumeId'): self.volume_id = res.get('VolumeId') return res
def _start(self): """ Calls boto3 create_file_system https://boto3.readthedocs.io/en/latest/reference/services/efs.html#EFS.Client.create_file_system Returns: boto3 create_file_system() response """ new_desired_state_def, diff_dict = pcf_util.update_dict( self.current_state_definition, self.get_desired_state_definition()) create_definition = pcf_util.param_filter(new_desired_state_def, EFS.START_PARAM_FILTER) res = self.client.create_file_system(**create_definition) self.client.create_tags(FileSystemId=res['FileSystemId'], Tags=[{ 'Key': 'Name', 'Value': self.instance_name }]) return res
def _update(self): """ Updates the lambda function particle to match current state definition. There is a different function call if the update is the function code in the zipfile or a configuration variable. """ if self.desired_state_definition["CodeSha256"] != self.current_state_definition["CodeSha256"]: if self.is_zip_local: self.client.update_function_code(FunctionName=self.function_name, ZipFile=self.file_get_contents()) else: self.client.update_function_code(FunctionName=self.function_name, **self.desired_state_definition["Code"]) self.desired_state_definition["CodeSha256"] = self.__zipfile_to_sha256() # update lambda configuration other than the code new_desired_state_def, diff_dict = pcf_util.update_dict(self.current_state_definition, self.get_desired_state_definition()) update_definition = pcf_util.param_filter(new_desired_state_def, LambdaFunction.UPDATE_PARAM_FILTER) if diff_dict != {}: self.client.update_function_configuration(**update_definition)
def test_apply(definition, changes, test_type): flavor = definition.get("flavor") particle_class = particle_flavor_scanner.get_particle_flavor(flavor) session = None with ExitStack() as stack: if test_type[0] == "placebo": session = boto3.Session() dirname = os.path.dirname(__file__) filename = os.path.join(dirname, test_type[1]) pill = placebo.attach(session, data_path=filename) pill.playback() else: for context in test_type: stack.enter_context(getattr(moto, context)()) # create particle = particle_class(definition, session) particle.set_desired_state(State.running) particle.apply(sync=True) assert particle.get_state() == State.running # print(particle.current_state_definition, particle.desired_state_definition) assert particle.is_state_definition_equivalent() # update if changes: updated_definition, diff = pcf_util.update_dict( definition, changes) if changes.get("aws_resource", {}).get("Tags"): updated_definition["aws_resource"]["Tags"] = changes.get( "aws_resource", {}).get("Tags") elif changes.get("aws_resource", {}).get("tags"): updated_definition["aws_resource"]["tags"] = changes.get( "aws_resource", {}).get("tags") particle = particle_class(updated_definition, session) particle.set_desired_state(State.running) particle.apply(sync=True) assert particle.is_state_definition_equivalent() # terminate particle.set_desired_state(State.terminated) particle.apply(sync=True) assert particle.get_state() == State.terminated
def _update(self): """ Updates the dynamodb particle to match current state definition. """ new_desired_state_def, diff_dict = pcf_util.update_dict( self.current_state_definition, self.get_desired_state_definition()) if diff_dict != {}: new_desired_state_def[ "ProvisionedThroughput"] = pcf_util.param_filter( new_desired_state_def["ProvisionedThroughput"], DynamoDB.THROUGHPUT_PARAM_FILTER) update_definition = pcf_util.param_filter( new_desired_state_def, DynamoDB.UPDATE_PARAM_FILTER) self.client.update_table(**update_definition) # compare tags if self._arn: table_arn = self._arn current_tags = self.client.list_tags_of_resource( ResourceArn=table_arn)["Tags"] else: table_arn = self.current_state_definition.get("TableArn") current_tags = self.client.list_tags_of_resource( ResourceArn=table_arn)["Tags"] desired_tags = self.desired_state_definition.get("Tags", []) if self._need_update(current_tags, desired_tags): add = list( itertools.filterfalse(lambda x: x in current_tags, desired_tags)) remove = list( itertools.filterfalse(lambda x: x in desired_tags, current_tags)) if remove: self.client.untag_resource( ResourceArn=table_arn, TagKeys=[x.get('Key') for x in remove]) if add: self.client.tag_resource(ResourceArn=table_arn, Tags=list(add))
def run_placebo(definition, updated_def, placebo_conf, action="record"): session = boto3.Session() dirname = os.path.dirname(__file__) filename = os.path.join(dirname, placebo_conf[1]) pill = placebo.attach(session, data_path=filename) if action == "playback": pill.playback() else: pill.record() flavor = definition.get("flavor") particle_class = particle_class_from_flavor(flavor) particle = particle_class(definition, session) # Test start particle.set_desired_state(State.running) particle.apply(sync=True) print(particle.get_state() == State.running) print(particle.is_state_definition_equivalent()) # Test update if updated_def: updated_definition, _ = update_dict(definition, updated_def) particle = particle_class(updated_definition, session) particle.set_desired_state(State.running) particle.apply(sync=True) print(particle.is_state_definition_equivalent()) # Test Terminate particle.set_desired_state(State.terminated) particle.apply(sync=True) print(particle.get_state() == State.terminated) pill.stop()
def test_apply(definition, updated_definition, test_type): flavor = definition.get("flavor") particle_class = particle_flavor_scanner.get_particle_flavor(flavor) session = None with ExitStack() as stack: if test_type[0] == "placebo": session = boto3.Session() dirname = os.path.dirname(__file__) filename = os.path.join(dirname, test_type[1]) pill = placebo.attach(session, data_path=filename) pill.playback() else: for context in test_type: stack.enter_context(getattr(moto, context)()) particle = particle_class(definition, session) particle.set_desired_state(State.running) particle.apply() assert particle.get_state() == State.running assert particle.is_state_definition_equivalent() if updated_definition: updated_definition, diff = pcf_util.update_dict( definition, updated_definition) print(updated_definition) particle = particle_class(updated_definition) particle.set_desired_state(State.running) particle.apply() assert particle.is_state_definition_equivalent() particle.set_desired_state(State.terminated) particle.apply() assert particle.get_state() == State.terminated
def generate_definition(self): self.particle.sync_state() # TODO generic for all resources # TODO desired definition is not always the same format as the current_definition self.particle_json["aws_resource"], _ = update_dict(self.particle.desired_state_definition,self.particle.current_state_definition) return self.particle_json
def test_update_dict_3(): orig_dict = { "gg": "wp", "lol": { "cats": "grumpy" }, "borg": ["resistance"], "don't": "touch me" } updated_dict = { "gg": "glhf", "lol": { "cats": "grumpy", "rofl": "copter" }, "borg": ["resistance", "is", "futile"] } diff_dict = pcf_util.diff_dict(orig_dict, updated_dict) assert (orig_dict == { "gg": "wp", "lol": { "cats": "grumpy" }, "borg": ["resistance"], "don't": "touch me" }) assert (diff_dict == { "borg": { "original": ["resistance"], "updated": ["resistance", "is", "futile"] }, "lol": { "rofl": { "new": "copter" } }, "gg": { "original": "wp", "updated": "glhf" } }) new_dict, diff_dict = pcf_util.update_dict(orig_dict, updated_dict) assert (new_dict == { "gg": "glhf", "lol": { "cats": "grumpy", "rofl": "copter" }, "borg": ["resistance", "is", "futile"], "don't": "touch me" }) assert (diff_dict == { "borg": { "original": ["resistance"], "updated": ["resistance", "is", "futile"] }, "lol": { "rofl": { "new": "copter" } }, "gg": { "original": "wp", "updated": "glhf" } })
def test_troll_update_dict_3(): orig_dict = {"lolcats": {"eat": "freefood"}} updated_dict = {} new_dict, diff_dict = pcf_util.update_dict(orig_dict, updated_dict) assert (new_dict == orig_dict) assert (diff_dict == {})