def _handle_first_request(self, parsed, primary_result_key,
starting_truncation):
# If the payload is an array or string, we need to slice into it
# and only return the truncated amount.
starting_truncation = self._parse_starting_token()[1]
all_data = primary_result_key.search(parsed)
if isinstance(all_data, (list, string_types)):
data = all_data[starting_truncation:]
else:
data = None
set_value_from_jmespath(
parsed,
primary_result_key.expression,
data
)
# We also need to truncate any secondary result keys
# because they were not truncated in the previous last
# response.
for token in self.result_keys:
if token == primary_result_key:
continue
sample = token.search(parsed)
if isinstance(sample, list):
empty_value = []
elif isinstance(sample, string_types):
empty_value = ''
elif isinstance(sample, (int, float)):
empty_value = 0
else:
empty_value = None
set_value_from_jmespath(parsed, token.expression, empty_value)
return starting_truncation
def _handle_first_request(self, parsed, primary_result_key,
starting_truncation):
# If the payload is an array or string, we need to slice into it
# and only return the truncated amount.
starting_truncation = self._parse_starting_token()[1]
all_data = primary_result_key.search(parsed)
if isinstance(all_data, (list, string_types)):
data = all_data[starting_truncation:]
else:
data = None
set_value_from_jmespath(parsed, primary_result_key.expression, data)
# We also need to truncate any secondary result keys
# because they were not truncated in the previous last
# response.
for token in self.result_keys:
if token == primary_result_key:
continue
sample = token.search(parsed)
if isinstance(sample, list):
empty_value = []
elif isinstance(sample, string_types):
empty_value = ''
elif isinstance(sample, (int, float)):
empty_value = 0
else:
empty_value = None
set_value_from_jmespath(parsed, token.expression, empty_value)
return starting_truncation
def __call__(self, command_name, response, stream=None):
if stream is None:
stream = self._get_default_stream()
try:
if is_response_paginated(response):
result_keys = response.result_keys
for i, page in enumerate(response):
if i > 0:
current = {}
else:
current = response.non_aggregate_part
for result_key in result_keys:
data = result_key.search(page)
set_value_from_jmespath(current, result_key.expression,
data)
self._format_response(current, stream)
if response.resume_token:
# Tell the user about the next token so they can continue
# if they want.
self._format_response(
{'NextToken': {
'NextToken': response.resume_token
}}, stream)
else:
self._remove_request_id(response)
self._format_response(response, stream)
finally:
# flush is needed to avoid the "close failed in file object
# destructor" in python2.x (see http://bugs.python.org/issue11380).
self._flush_stream(stream)
def do_export(self, resource_id, resource_dict, parent_dir):
"""
Default export action is to upload artifacts and set the property to
S3 URL of the uploaded object
If code signing configuration is provided for function/layer, uploaded artifact
will be replaced by signed artifact location
"""
# code signer only accepts files which has '.zip' extension in it
# so package artifact with '.zip' if it is required to be signed
should_sign_package = self.code_signer.should_sign_package(resource_id)
artifact_extension = "zip" if should_sign_package else None
uploaded_url = upload_local_artifacts(
resource_id,
resource_dict,
self.PROPERTY_NAME,
parent_dir,
self.uploader,
artifact_extension,
)
if should_sign_package:
uploaded_url = self.code_signer.sign_package(
resource_id, uploaded_url,
self.uploader.get_version_of_artifact(uploaded_url))
set_value_from_jmespath(resource_dict, self.PROPERTY_NAME,
uploaded_url)
def export(self, resource_id, resource_dict, parent_dir):
if resource_dict is None:
return
if resource_not_packageable(resource_dict):
return
property_value = jmespath.search(self.PROPERTY_NAME, resource_dict)
if not property_value and not self.PACKAGE_NULL_PROPERTY:
return
if isinstance(property_value, dict):
LOG.debug("Property %s of %s resource is not a URL", self.PROPERTY_NAME, resource_id)
return
# If property is a file but not a zip file, place file in temp
# folder and send the temp folder to be zipped
temp_dir = None
if is_local_file(property_value) and not is_zip_file(property_value) and self.FORCE_ZIP:
temp_dir = copy_to_temp_dir(property_value)
set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, temp_dir)
try:
self.do_export(resource_id, resource_dict, parent_dir)
except Exception as ex:
LOG.debug("Unable to export", exc_info=ex)
raise exceptions.ExportFailedError(
resource_id=resource_id, property_name=self.PROPERTY_NAME, property_value=property_value, ex=ex
)
finally:
if temp_dir:
shutil.rmtree(temp_dir)
def do_export(self, resource_id, resource_dict, parent_dir):
"""
Default export action is to upload artifacts and set the property to
S3 URL of the uploaded object
"""
uploaded_url = upload_local_artifacts(resource_id, resource_dict, self.PROPERTY_NAME, parent_dir, self.uploader)
set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, uploaded_url)
def build_full_result(self):
complete_result = {}
# Prepopulate the result keys with an empty list.
for result_expression in self.result_keys:
set_value_from_jmespath(complete_result,
result_expression.expression, [])
for response in self:
page = response
# We want to try to catch operation object pagination
# and format correctly for those. They come in the form
# of a tuple of two elements: (http_response, parsed_responsed).
# We want the parsed_response as that is what the page iterator
# uses. We can remove it though once operation objects are removed.
if isinstance(response, tuple) and len(response) == 2:
page = response[1]
# We're incrementally building the full response page
# by page. For each page in the response we need to
# inject the necessary components from the page
# into the complete_result.
for result_expression in self.result_keys:
# In order to incrementally update a result key
# we need to search the existing value from complete_result,
# then we need to search the _current_ page for the
# current result key value. Then we append the current
# value onto the existing value, and re-set that value
# as the new value.
existing_value = result_expression.search(complete_result)
result_value = result_expression.search(page)
if result_value is not None:
existing_value.extend(result_value)
merge_dicts(complete_result, self.non_aggregate_part)
if self.resume_token is not None:
complete_result['NextToken'] = self.resume_token
return complete_result
def build_full_result(self):
complete_result = {}
# Prepopulate the result keys with an empty list.
for result_expression in self.result_keys:
set_value_from_jmespath(complete_result,
result_expression.expression, [])
for _, page in self:
# We're incrementally building the full response page
# by page. For each page in the response we need to
# inject the necessary components from the page
# into the complete_result.
for result_expression in self.result_keys:
# In order to incrementally update a result key
# we need to search the existing value from complete_result,
# then we need to search the _current_ page for the
# current result key value. Then we append the current
# value onto the existing value, and re-set that value
# as the new value.
existing_value = result_expression.search(complete_result)
result_value = result_expression.search(page)
if result_value is not None:
existing_value.extend(result_value)
merge_dicts(complete_result, self.non_aggregate_part)
if self.resume_token is not None:
complete_result['NextToken'] = self.resume_token
return complete_result
def _record_non_aggregate_key_values(self, response):
non_aggregate_keys = {}
for expression in self._non_aggregate_key_exprs:
result = expression.search(response)
set_value_from_jmespath(non_aggregate_keys, expression.expression,
result)
self._non_aggregate_part = non_aggregate_keys
def build_full_result(self):
complete_result = {}
# Prepopulate the result keys with an empty list.
for result_expression in self.result_keys:
set_value_from_jmespath(complete_result,
result_expression.expression, [])
for response in self:
page = response
# We want to try to catch operation object pagination
# and format correctly for those. They come in the form
# of a tuple of two elements: (http_response, parsed_responsed).
# We want the parsed_response as that is what the page iterator
# uses. We can remove it though once operation objects are removed.
if isinstance(response, tuple) and len(response) == 2:
page = response[1]
# We're incrementally building the full response page
# by page. For each page in the response we need to
# inject the necessary components from the page
# into the complete_result.
for result_expression in self.result_keys:
# In order to incrementally update a result key
# we need to search the existing value from complete_result,
# then we need to search the _current_ page for the
# current result key value. Then we append the current
# value onto the existing value, and re-set that value
# as the new value.
existing_value = result_expression.search(complete_result)
result_value = result_expression.search(page)
if result_value is not None:
existing_value.extend(result_value)
merge_dicts(complete_result, self.non_aggregate_part)
if self.resume_token is not None:
complete_result['NextToken'] = self.resume_token
return complete_result
def build_full_result(self):
complete_result = {}
# Prepopulate the result keys with an empty list.
for result_expression in self.result_keys:
set_value_from_jmespath(complete_result, result_expression.expression, [])
for _, page in self:
# We're incrementally building the full response page
# by page. For each page in the response we need to
# inject the necessary components from the page
# into the complete_result.
for result_expression in self.result_keys:
# In order to incrementally update a result key
# we need to search the existing value from complete_result,
# then we need to search the _current_ page for the
# current result key value. Then we append the current
# value onto the existing value, and re-set that value
# as the new value.
existing_value = result_expression.search(complete_result)
result_value = result_expression.search(page)
if result_value is not None:
existing_value.extend(result_value)
merge_dicts(complete_result, self.non_aggregate_part)
if self.resume_token is not None:
complete_result["NextToken"] = self.resume_token
return complete_result
def __call__(self, operation, response, stream=None):
if stream is None:
stream = self._get_default_stream()
try:
if operation.can_paginate and self._args.paginate:
result_keys = response.result_keys
for _, page in response:
current = {}
for result_key in result_keys:
data = result_key.search(page)
set_value_from_jmespath(
current,
result_key.expression,
data
)
self._format_response(current, stream)
if response.resume_token:
# Tell the user about the next token so they can continue
# if they want.
self._format_response(
{'NextToken': {'NextToken': response.resume_token}},
stream)
else:
self._remove_request_id(response)
self._format_response(response, stream)
finally:
# flush is needed to avoid the "close failed in file object
# destructor" in python2.x (see http://bugs.python.org/issue11380).
self._flush_stream(stream)
def _truncate_response(self, parsed, primary_result_key, truncate_amount,
starting_truncation, next_token):
original = primary_result_key.search(parsed)
if original is None:
original = []
amount_to_keep = len(original) - truncate_amount
truncated = original[:amount_to_keep]
set_value_from_jmespath(
parsed,
primary_result_key.expression,
truncated
)
# The issue here is that even though we know how much we've truncated
# we need to account for this globally including any starting
# left truncation. For example:
# Raw response: [0,1,2,3]
# Starting index: 1
# Max items: 1
# Starting left truncation: [1, 2, 3]
# End right truncation for max items: [1]
# However, even though we only kept 1, this is post
# left truncation so the next starting index should be 2, not 1
# (left_truncation + amount_to_keep).
next_token.append(str(amount_to_keep + starting_truncation))
self.resume_token = next_token
def _record_non_aggregate_key_values(self, response):
non_aggregate_keys = {}
for expression in self._non_aggregate_key_exprs:
result = expression.search(response)
set_value_from_jmespath(non_aggregate_keys,
expression.expression,
result)
self._non_aggregate_part = non_aggregate_keys
def do_export(self, resource_id, resource_dict, parent_dir):
"""
Default export action is to upload artifacts and set the property to
dictionary where the key is EXPORT_PROPERTY_CODE_KEY and value is set to an
uploaded URL.
"""
uploaded_url = upload_local_image_artifacts(resource_id, resource_dict,
self.PROPERTY_NAME,
parent_dir, self.uploader)
set_value_from_jmespath(resource_dict, self.PROPERTY_NAME,
{self.EXPORT_PROPERTY_CODE_KEY: uploaded_url})
def _handle_first_request(self, parsed, primary_result_key, starting_truncation):
# First we need to slice into the array and only return
# the truncated amount.
starting_truncation = self._parse_starting_token()[1]
all_data = primary_result_key.search(parsed)
set_value_from_jmespath(parsed, primary_result_key.expression, all_data[starting_truncation:])
# We also need to truncate any secondary result keys
# because they were not truncated in the previous last
# response.
for token in self.result_keys:
if token == primary_result_key:
continue
set_value_from_jmespath(parsed, token.expression, [])
return starting_truncation
def _handle_first_request(self, parsed, primary_result_key,
starting_truncation):
# First we need to slice into the array and only return
# the truncated amount.
starting_truncation = self._parse_starting_token()[1]
all_data = primary_result_key.search(parsed)
set_value_from_jmespath(parsed, primary_result_key.expression,
all_data[starting_truncation:])
# We also need to truncate any secondary result keys
# because they were not truncated in the previous last
# response.
for token in self.result_keys:
if token == primary_result_key:
continue
set_value_from_jmespath(parsed, token.expression, [])
return starting_truncation
def build_full_result(self):
complete_result = {}
while True:
response = yield from self.next_page()
if response is None:
break
page = response
# We want to try to catch operation object pagination
# and format correctly for those. They come in the form
# of a tuple of two elements: (http_response, parsed_responsed).
# We want the parsed_response as that is what the page iterator
# uses. We can remove it though once operation objects are removed.
if isinstance(response, tuple) and len(response) == 2:
page = response[1]
# We're incrementally building the full response page
# by page. For each page in the response we need to
# inject the necessary components from the page
# into the complete_result.
for result_expression in self.result_keys:
# In order to incrementally update a result key
# we need to search the existing value from complete_result,
# then we need to search the _current_ page for the
# current result key value. Then we append the current
# value onto the existing value, and re-set that value
# as the new value.
result_value = result_expression.search(page)
if result_value is None:
continue
existing_value = result_expression.search(complete_result)
if existing_value is None:
# Set the initial result
set_value_from_jmespath(
complete_result, result_expression.expression,
result_value)
continue
# Now both result_value and existing_value contain something
if isinstance(result_value, list):
existing_value.extend(result_value)
elif isinstance(result_value, (int, float, str)):
# Modify the existing result with the sum or concatenation
set_value_from_jmespath(
complete_result, result_expression.expression,
existing_value + result_value)
merge_dicts(complete_result, self.non_aggregate_part)
if self.resume_token is not None:
complete_result['NextToken'] = self.resume_token
return complete_result
def build_full_result(self):
complete_result = {}
while True:
response = yield from self.next_page()
if response is None:
break
page = response
# We want to try to catch operation object pagination
# and format correctly for those. They come in the form
# of a tuple of two elements: (http_response, parsed_responsed).
# We want the parsed_response as that is what the page iterator
# uses. We can remove it though once operation objects are removed.
if isinstance(response, tuple) and len(response) == 2:
page = response[1]
# We're incrementally building the full response page
# by page. For each page in the response we need to
# inject the necessary components from the page
# into the complete_result.
for result_expression in self.result_keys:
# In order to incrementally update a result key
# we need to search the existing value from complete_result,
# then we need to search the _current_ page for the
# current result key value. Then we append the current
# value onto the existing value, and re-set that value
# as the new value.
result_value = result_expression.search(page)
if result_value is None:
continue
existing_value = result_expression.search(complete_result)
if existing_value is None:
# Set the initial result
set_value_from_jmespath(complete_result,
result_expression.expression,
result_value)
continue
# Now both result_value and existing_value contain something
if isinstance(result_value, list):
existing_value.extend(result_value)
elif isinstance(result_value, (int, float, str)):
# Modify the existing result with the sum or concatenation
set_value_from_jmespath(complete_result,
result_expression.expression,
existing_value + result_value)
merge_dicts(complete_result, self.non_aggregate_part)
if self.resume_token is not None:
complete_result['NextToken'] = self.resume_token
return complete_result
def do_export(self, resource_id, resource_dict, parent_dir):
"""
Upload to S3 and set property to an dict representing the S3 url
of the uploaded object
"""
artifact_s3_url = \
upload_local_artifacts(resource_id, resource_dict,
self.PROPERTY_NAME,
parent_dir, self.uploader)
parsed_url = parse_s3_url(
artifact_s3_url,
bucket_name_property=self.BUCKET_NAME_PROPERTY,
object_key_property=self.OBJECT_KEY_PROPERTY,
version_property=self.VERSION_PROPERTY)
set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, parsed_url)
def do_export(self, resource_id, resource_dict, parent_dir):
"""
If the nested stack template is valid, this method will
export on the nested template, upload the exported template to S3
and set property to URL of the uploaded S3 template
"""
template_path = resource_dict.get(self.PROPERTY_NAME, None)
if (template_path is None or is_s3_url(template_path)
or template_path.startswith(self.uploader.s3.meta.endpoint_url)
or template_path.startswith("https://s3.amazonaws.com/")):
# Nothing to do
return
abs_template_path = make_abs_path(parent_dir, template_path)
if not is_local_file(abs_template_path):
raise exceptions.InvalidTemplateUrlParameterError(
property_name=self.PROPERTY_NAME,
resource_id=resource_id,
template_path=abs_template_path)
exported_template_dict = Template(template_path, parent_dir,
self.uploaders,
self.code_signer).export()
exported_template_str = yaml_dump(exported_template_dict)
with mktempfile() as temporary_file:
temporary_file.write(exported_template_str)
temporary_file.flush()
url = self.uploader.upload_with_dedup(temporary_file.name,
"template")
# TemplateUrl property requires S3 URL to be in path-style format
parts = S3Uploader.parse_s3_url(url, version_property="Version")
s3_path_url = self.uploader.to_path_style_s3_url(
parts["Key"], parts.get("Version", None))
set_value_from_jmespath(resource_dict, self.PROPERTY_NAME,
s3_path_url)
def build_full_result(self):
iterators = self.result_key_iters()
response = {}
key_names = [i.result_key for i in iterators]
for key in key_names:
set_value_from_jmespath(response, key.expression, [])
for vals in zip_longest(*iterators):
for k, val in zip(key_names, vals):
if val is not None:
# We can't just do an append here, since we don't
# necessarily have a reference (& the expression isn't as
# easy as normal keys).
# So we'll search for the value, create an empty list if
# we don't find it, then append the ``val`` to the list &
# set it via the expression.
existing = k.search(response)
if not existing:
existing = []
existing.append(val)
set_value_from_jmespath(response, k.expression, existing)
if self.resume_token is not None:
response['NextToken'] = self.resume_token
return response
def test_must_update_relative_resource_paths(self, resource_type, properties):
for propname in properties:
template_dict = {
"Resources": {
"MyResourceWithRelativePath": {"Type": resource_type, "Properties": {}},
"MyResourceWithS3Path": {"Type": resource_type, "Properties": {propname: self.s3path}},
"MyResourceWithAbsolutePath": {"Type": resource_type, "Properties": {propname: self.abspath}},
"MyResourceWithInvalidPath": {
"Type": resource_type,
"Properties": {
# Path is not a string
propname: {"foo": "bar"}
},
},
"MyResourceWithoutProperties": {"Type": resource_type},
"UnsupportedResourceType": {"Type": "AWS::Ec2::Instance", "Properties": {"Code": "bar"}},
"ResourceWithoutType": {"foo": "bar"},
},
"Parameters": {"a": "b"},
}
set_value_from_jmespath(
template_dict, f"Resources.MyResourceWithRelativePath.Properties.{propname}", self.curpath
)
expected_template_dict = copy.deepcopy(template_dict)
set_value_from_jmespath(
expected_template_dict,
f"Resources.MyResourceWithRelativePath.Properties.{propname}",
self.expected_result,
)
result = _update_relative_paths(template_dict, self.src, self.dest)
self.maxDiff = None
self.assertEqual(result, expected_template_dict)
def test_multiple_depth_new(self):
self.assertFalse('Brand' in self.data)
set_value_from_jmespath(self.data, 'Brand.New', {'abc': 123})
self.assertEqual(self.data['Brand']['New']['abc'], 123)
def test_multiple_depth_new(self):
self.assertFalse('Brand' in self.data)
set_value_from_jmespath(self.data, 'Brand.New', {'abc': 123})
self.assertEqual(self.data['Brand']['New']['abc'], 123)
def test_multiple_depth_existing(self):
set_value_from_jmespath(self.data, 'Response.Thing.Name', 'New Name')
self.assertEqual(self.data['Response']['Thing']['Name'], 'New Name')
def test_single_depth_new(self):
self.assertFalse('Limit' in self.data)
set_value_from_jmespath(self.data, 'Limit', 100)
self.assertEqual(self.data['Limit'], 100)
def test_single_depth_existing(self):
set_value_from_jmespath(self.data, 'Marker', 'new-token')
self.assertEqual(self.data['Marker'], 'new-token')
def test_single_depth_existing(self):
set_value_from_jmespath(self.data, 'Marker', 'new-token')
self.assertEqual(self.data['Marker'], 'new-token')
def test_multiple_depth_existing(self):
set_value_from_jmespath(self.data, "Response.Thing.Name", "New Name")
self.assertEqual(self.data["Response"]["Thing"]["Name"], "New Name")
def test_multiple_depth_new(self):
self.assertFalse("Brand" in self.data)
set_value_from_jmespath(self.data, "Brand.New", {"abc": 123})
self.assertEqual(self.data["Brand"]["New"]["abc"], 123)
def test_single_depth_new(self):
self.assertFalse("Limit" in self.data)
set_value_from_jmespath(self.data, "Limit", 100)
self.assertEqual(self.data["Limit"], 100)
def test_single_depth_existing(self):
set_value_from_jmespath(self.data, "Marker", "new-token")
self.assertEqual(self.data["Marker"], "new-token")
def test_single_depth_new(self):
self.assertFalse('Limit' in self.data)
set_value_from_jmespath(self.data, 'Limit', 100)
self.assertEqual(self.data['Limit'], 100)
def test_multiple_depth_existing(self):
set_value_from_jmespath(self.data, 'Response.Thing.Name', 'New Name')
self.assertEqual(self.data['Response']['Thing']['Name'], 'New Name')
def _update_relative_paths(template_dict, original_root, new_root):
"""
SAM/CloudFormation template can contain certain properties whose value is a relative path to a local file/folder.
This path is usually relative to the template's location. If the template is being moved from original location
``original_root`` to new location ``new_root``, use this method to update these paths to be
relative to ``new_root``.
After this method is complete, it is safe to write the template to ``new_root`` without
breaking any relative paths.
This methods updates resource properties supported by ``aws cloudformation package`` command:
https://docs.aws.amazon.com/cli/latest/reference/cloudformation/package.html
If a property is either an absolute path or a S3 URI, this method will not update them.
Parameters
----------
template_dict : dict
Dictionary containing template contents. This dictionary will be updated & written to ``dest`` location.
original_root : str
Path to the directory where all paths were originally set relative to. This is usually the directory
containing the template originally
new_root : str
Path to the new directory that all paths set relative to after this method completes.
Returns
-------
Updated dictionary
"""
for resource_type, properties in template_dict.get("Metadata", {}).items():
if resource_type not in METADATA_WITH_LOCAL_PATHS:
# Unknown resource. Skipping
continue
for path_prop_name in METADATA_WITH_LOCAL_PATHS[resource_type]:
path = properties.get(path_prop_name)
updated_path = _resolve_relative_to(path, original_root, new_root)
if not updated_path:
# This path does not need to get updated
continue
properties[path_prop_name] = updated_path
for _, resource in template_dict.get("Resources", {}).items():
resource_type = resource.get("Type")
if resource_type not in RESOURCES_WITH_LOCAL_PATHS:
# Unknown resource. Skipping
continue
for path_prop_name in RESOURCES_WITH_LOCAL_PATHS[resource_type]:
properties = resource.get("Properties", {})
path = jmespath.search(path_prop_name, properties)
updated_path = _resolve_relative_to(path, original_root, new_root)
if not updated_path:
# This path does not need to get updated
continue
set_value_from_jmespath(properties, path_prop_name, updated_path)
# AWS::Includes can be anywhere within the template dictionary. Hence we need to recurse through the
# dictionary in a separate method to find and update relative paths in there
template_dict = _update_aws_include_relative_path(template_dict,
original_root, new_root)
return template_dict
