def test_update_dict():
orig_dict = {}
updated_dict = {
"gg": "wp",
"lol": {
"cats": "grumpy"
},
"borg": ["resistance", "is", "futile"]
}
new_dict, diff_dict = pcf_util.update_dict(orig_dict, updated_dict)
assert (orig_dict == {})
assert (new_dict == updated_dict)
assert (diff_dict == {
"gg": {
"new": "wp"
},
"lol": {
"new": {
"cats": "grumpy"
}
},
"borg": {
"new": ["resistance", "is", "futile"]
}
})
def _update(self):
"""
Calls boto3 put_attributes() to update ECS Instance attributes. Does not allow for attributes that start
with com.amazonaws.ecs. or instance-id to be updated.
Returns:
boto3 put_attributes() response
"""
new_desired_state_def, diff_dict = pcf_util.update_dict(
self.current_state_definition, self.get_desired_state_definition())
update_definition = pcf_util.param_filter(
new_desired_state_def, ECSInstance.UPDATE_PARAM_FILTER)
attributes = []
for a in update_definition['attributes']:
if (not a['name'].startswith('ecs.')
and not a['name'].startswith('com.amazonaws.ecs.')
and a['name'] not in ECSInstance.PROTECTED_ATTRIBUTES):
attributes.append({
'name': a['name'],
'value': a['value'],
'targetType': 'container-instance',
'targetId': self.get_ecs_instance_id(),
})
return self.client.put_attributes(
cluster=self.get_cluster_name(),
attributes=attributes,
)
def is_state_definition_equivalent(self):
"""
Compares the desired state and current state definitions.
Returns:
bool
"""
self.get_state()
current_definition = pcf_util.param_filter(
self.current_state_definition, DynamoDB.REMOVE_PARAM_FILTER, True)
desired_definition = pcf_util.param_filter(
self.desired_state_definition, DynamoDB.START_PARAM_FILTER)
# compare tags
if self._arn:
current_tags = self.client.list_tags_of_resource(
ResourceArn=self._arn)["Tags"]
else:
current_tags = self.client.list_tags_of_resource(
ResourceArn=self.current_state_definition.get(
"TableArn"))["Tags"]
desired_tags = desired_definition.get("Tags", [])
new_desired_state_def, diff_dict = pcf_util.update_dict(
current_definition, desired_definition)
# self.create_table() does not return "Tags" as an attribute therefore, current_state_definition does not have
# any reference to Tags, so it is removed from the comparison between current_definition and desired_definition
diff_dict.pop('Tags', None)
return diff_dict == {} and not self._need_update(
current_tags, desired_tags)
def test_troll_update_string():
orig_dict = {"lolcats": "lol"}
updated_dict = {"lolcats": ""}
new_dict, diff_dict = pcf_util.update_dict(orig_dict, updated_dict)
assert (new_dict == updated_dict)
assert (diff_dict == {
'lolcats': {
'original': "lol",
'updated': "",
}
})
def test_troll_update_list():
orig_dict = {"lolcats": ["fly"]}
updated_dict = {"lolcats": []}
new_dict, diff_dict = pcf_util.update_dict(orig_dict, updated_dict)
assert (new_dict == updated_dict)
assert (diff_dict == {
'lolcats': {
'original': ["fly"],
'updated': [],
}
})
def is_state_definition_equivalent(self):
"""
Compares the desired state and current state definitions.
Returns:
bool
"""
self.get_state()
desired_definition = pcf_util.param_filter(self.desired_state_definition, LambdaFunction.RETURN_PARAM_FILTER)
new_desired_state_def, diff_dict = pcf_util.update_dict(self.current_state_definition, desired_definition)
return diff_dict == {}
def _start(self):
"""
Starts the lambda particle that matches desired state definition
:return: response of boto3 create_function
"""
new_desired_state_def, diff_dict = pcf_util.update_dict(self.current_state_definition, self.get_desired_state_definition())
start_definition = pcf_util.param_filter(new_desired_state_def, LambdaFunction.START_PARAM_FILTER)
if self.is_zip_local:
start_definition["Code"]["ZipFile"] = self.file_get_contents()
self.client.create_function(**start_definition)
def _update(self):
"""
Updates the dynamodb particle to match current state definition.
"""
new_desired_state_def, diff_dict = pcf_util.update_dict(
self.current_state_definition, self.get_desired_state_definition())
new_desired_state_def["ProvisionedThroughput"] = pcf_util.param_filter(
new_desired_state_def["ProvisionedThroughput"],
DynamoDB.THROUGHPUT_PARAM_FILTER)
update_definition = pcf_util.param_filter(new_desired_state_def,
DynamoDB.UPDATE_PARAM_FILTER)
self.client.update_table(**update_definition)
def _start(self):
"""
Calls boto3 register_task_definition()
Returns:
boto3 register_task_definition() response
"""
new_desired_state_def, diff_dict = pcf_util.update_dict(self.current_state_definition, self.get_desired_state_definition())
new_desired_state_def = pcf_util.param_filter(new_desired_state_def, ECSTaskDefinition.START_PARAM_FILTER)
resp = self.client.register_task_definition(**new_desired_state_def)
self.current_state_definition = resp
return resp
def test_troll_update_dict_2():
orig_dict = {"lolcats": {"eat": "freefood"}}
updated_dict = {"lolcats": None}
new_dict, diff_dict = pcf_util.update_dict(orig_dict, updated_dict)
assert (new_dict == updated_dict)
assert (diff_dict == {
'lolcats': {
'original': {
"eat": "freefood"
},
'updated': None,
}
})
def update_particle_definition(self, particle_definition, base_particle):
"""
Updates a particle's definition with a previous definition. Replaces any fields that are specified.
Args:
particle_definition (dict): new particle defintion (can be empty)
base_particle (particle): the particle that the new particle is inheriting from
Returns:
updated_particle_definition
"""
updated_particle_defintion, diff_dict = pcf_util.update_dict(base_particle.particle_definition, particle_definition)
return updated_particle_defintion
def _update(self):
"""
Calls boto3 update_service()
Returns:
boto3 update_service() response
"""
new_desired_state_def, diff_dict = pcf_util.update_dict(
self.current_state_definition, self.get_desired_state_definition())
update_definition = pcf_util.keep_and_replace_keys(
new_desired_state_def, ECSService.UPDATE_PARAM_CONVERSIONS)
return self.client.update_service(**update_definition)
def is_state_definition_equivalent(self):
"""
Compares the desired state and current state definitions.
Returns:
bool
"""
self.get_state()
current_definition = pcf_util.param_filter(
self.current_state_definition, DynamoDB.REMOVE_PARAM_FILTER, True)
desired_definition = pcf_util.param_filter(
self.desired_state_definition, DynamoDB.START_PARAM_FILTER)
new_desired_state_def, diff_dict = pcf_util.update_dict(
current_definition, desired_definition)
return diff_dict == {}
def _start(self):
"""
Calls boto3 run_task function to create a new task. If successful this gets the arn of the task and adds it to the
current_state_definition.
"""
new_desired_state_def, diff_dict = pcf_util.update_dict(
self.current_state_definition, self.get_desired_state_definition())
start_definition = pcf_util.keep_and_replace_keys(
new_desired_state_def, ECSTask.START_PARAM_CONVERSIONS)
self.sync_state()
if self.state == State.stopped and "containers" in self.current_state_definition:
containers = self.current_state_definition.get("containers")
for container in containers:
if container.get("exitCode", -1) != 0:
logger.warning(
"Task {} failed to execute with exit code: {} and reason: {}... setting desired state to {}"
.format(self.get_task_arn(), container.get("exitCode"),
container.get("reason"), State.stopped))
self.failure_reason = {
"type":
"container",
"reason":
self.current_state_definition.get(
"stoppedReason", "N/A")
}
self.set_desired_state(State.stopped)
return
resp = self.client.run_task(**start_definition)
task = resp.get("tasks", [])
failures = resp.get("failures", [])
if len(task) == 1:
self.current_state_definition["taskArn"] = task[0].get("taskArn")
elif len(task) == 0 and len(failures) > 0:
logger.warning(
"Task {} failed to be placed due to an ECS error: {}... setting desired state to {}"
.format(self.get_task_arn(), failures[0].get("reason"),
State.stopped))
self.failure_reason = {
"type": "ecs",
"reason": failures[0].get("reason")
}
self.set_desired_state(State.stopped)
else:
Exception("ECS Task failed to start")
def _start(self):
"""
Calls boto3 create_volume().
Returns:
boto3 create_volume() response
"""
new_desired_state_def, diff_dict = pcf_util.update_dict(
self.current_state_definition, self.get_desired_state_definition())
start_params = pcf_util.param_filter(new_desired_state_def,
EBSVolume.START_PARAM_FILTER)
res = self.client.create_volume(**start_params)
if res.get('VolumeId'):
self.volume_id = res.get('VolumeId')
return res
def _start(self):
"""
Calls boto3 create_file_system https://boto3.readthedocs.io/en/latest/reference/services/efs.html#EFS.Client.create_file_system
Returns:
boto3 create_file_system() response
"""
new_desired_state_def, diff_dict = pcf_util.update_dict(
self.current_state_definition, self.get_desired_state_definition())
create_definition = pcf_util.param_filter(new_desired_state_def,
EFS.START_PARAM_FILTER)
res = self.client.create_file_system(**create_definition)
self.client.create_tags(FileSystemId=res['FileSystemId'],
Tags=[{
'Key': 'Name',
'Value': self.instance_name
}])
return res
def _update(self):
"""
Updates the lambda function particle to match current state definition. There is a different
function call if the update is the function code in the zipfile or a configuration variable.
"""
if self.desired_state_definition["CodeSha256"] != self.current_state_definition["CodeSha256"]:
if self.is_zip_local:
self.client.update_function_code(FunctionName=self.function_name, ZipFile=self.file_get_contents())
else:
self.client.update_function_code(FunctionName=self.function_name, **self.desired_state_definition["Code"])
self.desired_state_definition["CodeSha256"] = self.__zipfile_to_sha256()
# update lambda configuration other than the code
new_desired_state_def, diff_dict = pcf_util.update_dict(self.current_state_definition, self.get_desired_state_definition())
update_definition = pcf_util.param_filter(new_desired_state_def, LambdaFunction.UPDATE_PARAM_FILTER)
if diff_dict != {}:
self.client.update_function_configuration(**update_definition)
def test_apply(definition, changes, test_type):
flavor = definition.get("flavor")
particle_class = particle_flavor_scanner.get_particle_flavor(flavor)
session = None
with ExitStack() as stack:
if test_type[0] == "placebo":
session = boto3.Session()
dirname = os.path.dirname(__file__)
filename = os.path.join(dirname, test_type[1])
pill = placebo.attach(session, data_path=filename)
pill.playback()
else:
for context in test_type:
stack.enter_context(getattr(moto, context)())
# create
particle = particle_class(definition, session)
particle.set_desired_state(State.running)
particle.apply(sync=True)
assert particle.get_state() == State.running
# print(particle.current_state_definition, particle.desired_state_definition)
assert particle.is_state_definition_equivalent()
# update
if changes:
updated_definition, diff = pcf_util.update_dict(
definition, changes)
if changes.get("aws_resource", {}).get("Tags"):
updated_definition["aws_resource"]["Tags"] = changes.get(
"aws_resource", {}).get("Tags")
elif changes.get("aws_resource", {}).get("tags"):
updated_definition["aws_resource"]["tags"] = changes.get(
"aws_resource", {}).get("tags")
particle = particle_class(updated_definition, session)
particle.set_desired_state(State.running)
particle.apply(sync=True)
assert particle.is_state_definition_equivalent()
# terminate
particle.set_desired_state(State.terminated)
particle.apply(sync=True)
assert particle.get_state() == State.terminated
def _update(self):
"""
Updates the dynamodb particle to match current state definition.
"""
new_desired_state_def, diff_dict = pcf_util.update_dict(
self.current_state_definition, self.get_desired_state_definition())
if diff_dict != {}:
new_desired_state_def[
"ProvisionedThroughput"] = pcf_util.param_filter(
new_desired_state_def["ProvisionedThroughput"],
DynamoDB.THROUGHPUT_PARAM_FILTER)
update_definition = pcf_util.param_filter(
new_desired_state_def, DynamoDB.UPDATE_PARAM_FILTER)
self.client.update_table(**update_definition)
# compare tags
if self._arn:
table_arn = self._arn
current_tags = self.client.list_tags_of_resource(
ResourceArn=table_arn)["Tags"]
else:
table_arn = self.current_state_definition.get("TableArn")
current_tags = self.client.list_tags_of_resource(
ResourceArn=table_arn)["Tags"]
desired_tags = self.desired_state_definition.get("Tags", [])
if self._need_update(current_tags, desired_tags):
add = list(
itertools.filterfalse(lambda x: x in current_tags,
desired_tags))
remove = list(
itertools.filterfalse(lambda x: x in desired_tags,
current_tags))
if remove:
self.client.untag_resource(
ResourceArn=table_arn,
TagKeys=[x.get('Key') for x in remove])
if add:
self.client.tag_resource(ResourceArn=table_arn, Tags=list(add))
def run_placebo(definition, updated_def, placebo_conf, action="record"):
session = boto3.Session()
dirname = os.path.dirname(__file__)
filename = os.path.join(dirname, placebo_conf[1])
pill = placebo.attach(session, data_path=filename)
if action == "playback":
pill.playback()
else:
pill.record()
flavor = definition.get("flavor")
particle_class = particle_class_from_flavor(flavor)
particle = particle_class(definition, session)
# Test start
particle.set_desired_state(State.running)
particle.apply(sync=True)
print(particle.get_state() == State.running)
print(particle.is_state_definition_equivalent())
# Test update
if updated_def:
updated_definition, _ = update_dict(definition, updated_def)
particle = particle_class(updated_definition, session)
particle.set_desired_state(State.running)
particle.apply(sync=True)
print(particle.is_state_definition_equivalent())
# Test Terminate
particle.set_desired_state(State.terminated)
particle.apply(sync=True)
print(particle.get_state() == State.terminated)
pill.stop()
def test_apply(definition, updated_definition, test_type):
flavor = definition.get("flavor")
particle_class = particle_flavor_scanner.get_particle_flavor(flavor)
session = None
with ExitStack() as stack:
if test_type[0] == "placebo":
session = boto3.Session()
dirname = os.path.dirname(__file__)
filename = os.path.join(dirname, test_type[1])
pill = placebo.attach(session, data_path=filename)
pill.playback()
else:
for context in test_type:
stack.enter_context(getattr(moto, context)())
particle = particle_class(definition, session)
particle.set_desired_state(State.running)
particle.apply()
assert particle.get_state() == State.running
assert particle.is_state_definition_equivalent()
if updated_definition:
updated_definition, diff = pcf_util.update_dict(
definition, updated_definition)
print(updated_definition)
particle = particle_class(updated_definition)
particle.set_desired_state(State.running)
particle.apply()
assert particle.is_state_definition_equivalent()
particle.set_desired_state(State.terminated)
particle.apply()
assert particle.get_state() == State.terminated
def generate_definition(self):
self.particle.sync_state()
# TODO generic for all resources
# TODO desired definition is not always the same format as the current_definition
self.particle_json["aws_resource"], _ = update_dict(self.particle.desired_state_definition,self.particle.current_state_definition)
return self.particle_json
def test_update_dict_3():
orig_dict = {
"gg": "wp",
"lol": {
"cats": "grumpy"
},
"borg": ["resistance"],
"don't": "touch me"
}
updated_dict = {
"gg": "glhf",
"lol": {
"cats": "grumpy",
"rofl": "copter"
},
"borg": ["resistance", "is", "futile"]
}
diff_dict = pcf_util.diff_dict(orig_dict, updated_dict)
assert (orig_dict == {
"gg": "wp",
"lol": {
"cats": "grumpy"
},
"borg": ["resistance"],
"don't": "touch me"
})
assert (diff_dict == {
"borg": {
"original": ["resistance"],
"updated": ["resistance", "is", "futile"]
},
"lol": {
"rofl": {
"new": "copter"
}
},
"gg": {
"original": "wp",
"updated": "glhf"
}
})
new_dict, diff_dict = pcf_util.update_dict(orig_dict, updated_dict)
assert (new_dict == {
"gg": "glhf",
"lol": {
"cats": "grumpy",
"rofl": "copter"
},
"borg": ["resistance", "is", "futile"],
"don't": "touch me"
})
assert (diff_dict == {
"borg": {
"original": ["resistance"],
"updated": ["resistance", "is", "futile"]
},
"lol": {
"rofl": {
"new": "copter"
}
},
"gg": {
"original": "wp",
"updated": "glhf"
}
})
def test_troll_update_dict_3():
orig_dict = {"lolcats": {"eat": "freefood"}}
updated_dict = {}
new_dict, diff_dict = pcf_util.update_dict(orig_dict, updated_dict)
assert (new_dict == orig_dict)
assert (diff_dict == {})
