def imports(self):
def _collect(o, **kwargs):
if isinstance(o, dict):
import_val = o.get('Fn::ImportValue')
if import_val:
result.add(import_val)
return o
result = set()
recurse_object(self.resources, _collect)
return result
def render_velocity_template(template, context, variables={}, as_json=False):
import airspeed
# Apply a few fixes below, to properly prepare the template...
# fix "#set" commands
template = re.sub(r'(^|\n)#\s+set(.*)', r'\1#set\2', template,
re.MULTILINE)
# enable syntax like "test#${foo.bar}"
empty_placeholder = ' __pLaCe-HoLdEr__ '
template = re.sub(r'([^\s]+)#\$({)?(.*)',
r'\1#%s$\2\3' % empty_placeholder, template,
re.MULTILINE)
# add extensions for common string functions below
class ExtendedString(str):
def trim(self, *args, **kwargs):
return ExtendedString(self.strip(*args, **kwargs))
def toLowerCase(self, *args, **kwargs):
return ExtendedString(self.lower(*args, **kwargs))
def toUpperCase(self, *args, **kwargs):
return ExtendedString(self.upper(*args, **kwargs))
def apply(obj, **kwargs):
if isinstance(obj, dict):
for k, v in obj.items():
if isinstance(v, str):
obj[k] = ExtendedString(v)
return obj
# loop through the variables and enable certain additional util functions (e.g., string utils)
variables = variables or {}
recurse_object(variables, apply)
# prepare and render template
t = airspeed.Template(template)
var_map = {'input': VelocityInput(context), 'util': VelocityUtil()}
var_map.update(variables or {})
replaced = t.merge(var_map)
# revert temporary changes from the fixes above
replaced = replaced.replace(empty_placeholder, '')
if as_json:
replaced = json.loads(replaced)
return replaced
def itemize(data, parent_key=None, *args, **kwargs):
# TODO: potentially add additional required tags here!
list_parent_tags = ["ClusterSubnetGroups"]
def fix_keys(o, **kwargs):
if isinstance(o, dict):
for k, v in o.items():
if k in list_parent_tags:
if isinstance(v, dict) and "item" in v:
v[k[:-1]] = v.pop("item")
return o
result = itemize_orig(data, *args, **kwargs)
recurse_object(result, fix_keys)
return result
def events_put_rule_params(params, **kwargs):
attrs = [
"ScheduleExpression",
"EventPattern",
"State",
"Description",
"Name",
"EventBusName",
]
result = select_parameters(*attrs)(params, **kwargs)
# TODO: remove this when refactoring events (prefix etc. was excluded here already to avoid most of the wrong behavior)
def wrap_in_lists(o, **kwargs):
if isinstance(o, dict):
for k, v in o.items():
if not isinstance(v, (dict, list)) and k not in [
"prefix",
"cidr",
"exists",
]:
o[k] = [v]
return o
pattern = result.get("EventPattern")
if isinstance(pattern, dict):
wrapped = common.recurse_object(pattern, wrap_in_lists)
result["EventPattern"] = json.dumps(wrapped)
return result
def set_moto_account_ids(resource_json):
def fix_ids(obj, **kwargs):
if isinstance(obj, dict):
for key, value in obj.items():
if 'arn' in key.lower() and isinstance(value, six.string_types):
obj[key] = value.replace(TEST_AWS_ACCOUNT_ID, MOTO_ACCOUNT_ID)
return obj
return recurse_object(resource_json, fix_ids)
def fix_account_id_in_arns(params):
def fix_ids(o, **kwargs):
if isinstance(o, dict):
for k, v in o.items():
if common.is_string(v, exclude_binary=True):
o[k] = aws_stack.fix_account_id_in_arns(v)
return o
result = common.recurse_object(params, fix_ids)
return result
def remove_none_values(params):
""" Remove None values recursively in the given object. """
def remove_nones(o, **kwargs):
if isinstance(o, dict):
for k, v in dict(o).items():
if v is None:
o.pop(k)
return o
result = common.recurse_object(params, remove_nones)
return result
def convert_objs_to_ids(resource_json):
def fix_ids(obj, **kwargs):
if isinstance(obj, dict):
obj = dict(obj)
for key, value in obj.items():
if isinstance(value, BaseModel):
entity_id = get_entity_id(value)
obj[key] = entity_id or value
return obj
return recurse_object(resource_json, fix_ids)
def add_vpc_info_to_response(path: str, response: Response):
content = to_str(response.content or "")
if "<HostedZone>" not in content:
return
if "GetHostedZoneResponse" not in content and "CreateHostedZoneResponse" not in content:
return
content = clone(xmltodict.parse(content))
region_details = Route53Backend.get()
def _insert(obj, **_):
if not isinstance(obj, dict) or "HostedZone" not in obj or "VPCs" in obj:
return obj
zone_id = obj["HostedZone"].get("Id", "").replace("/hostedzone/", "")
zone_details = region_details.vpc_hosted_zone_associations.get(zone_id) or []
vpcs = [zone["VPC"] for zone in zone_details if zone.get("VPC")]
if vpcs:
obj["VPCs"] = [{"VPC": vpc} for vpc in vpcs]
return obj
recurse_object(content, _insert)
set_response_content(response, xmltodict.unparse(content))
def remove_none_values(params):
""" Remove None values recursively in the given object. """
def remove_nones(o, **kwargs):
if isinstance(o, dict):
for k, v in dict(o).items():
if v is None:
o.pop(k)
if isinstance(o, list):
common.run_safe(o.remove, None)
common.run_safe(o.remove, PLACEHOLDER_AWS_NO_VALUE)
return o
result = common.recurse_object(params, remove_nones)
return result
def convert_data_types(func_details, params):
""" Convert data types in the "params" object, with the type defs
specified in the 'types' attribute of "func_details". """
types = func_details.get('types') or {}
attr_names = types.keys() or []
def cast(_obj, _type):
if _type == bool:
return _obj in ['True', 'true', True]
if _type == str:
return str(_obj)
if _type == int:
return int(_obj)
return _obj
def fix_types(o, **kwargs):
if isinstance(o, dict):
for k, v in o.items():
if k in attr_names:
o[k] = cast(v, types[k])
return o
result = common.recurse_object(params, fix_types)
return result
def events_put_rule_params(params, **kwargs):
attrs = [
"ScheduleExpression",
"EventPattern",
"State",
"Description",
"Name",
"EventBusName",
]
result = select_parameters(*attrs)(params, **kwargs)
def wrap_in_lists(o, **kwargs):
if isinstance(o, dict):
for k, v in o.items():
if not isinstance(v, (dict, list)):
o[k] = [v]
return o
pattern = result.get("EventPattern")
if isinstance(pattern, dict):
wrapped = common.recurse_object(pattern, wrap_in_lists)
result["EventPattern"] = json.dumps(wrapped)
return result
def configure_resource_via_sdk(resource_id, resources, resource_type, func_details, stack_name):
resource = resources[resource_id]
client = get_client(resource, func_details)
function = getattr(client, func_details['function'])
params = func_details.get('parameters') or lambda_get_params()
defaults = func_details.get('defaults', {})
if 'Properties' not in resource:
resource['Properties'] = {}
resource_props = resource['Properties']
if callable(params):
params = params(resource_props, stack_name=stack_name, resources=resources)
else:
params = dict(params)
for param_key, prop_keys in dict(params).items():
params.pop(param_key, None)
if not isinstance(prop_keys, list):
prop_keys = [prop_keys]
for prop_key in prop_keys:
if prop_key == PLACEHOLDER_RESOURCE_NAME:
params[param_key] = PLACEHOLDER_RESOURCE_NAME
else:
if callable(prop_key):
prop_value = prop_key(resource_props, stack_name=stack_name, resources=resources)
else:
prop_value = resource_props.get(prop_key)
if prop_value is not None:
params[param_key] = prop_value
break
# replace PLACEHOLDER_RESOURCE_NAME in params
resource_name_holder = {}
def fix_placeholders(o, **kwargs):
if isinstance(o, dict):
for k, v in o.items():
if v == PLACEHOLDER_RESOURCE_NAME:
if 'value' not in resource_name_holder:
resource_name_holder['value'] = get_resource_name(resource) or resource_id
o[k] = resource_name_holder['value']
return o
common.recurse_object(params, fix_placeholders)
# assign default values if empty
params = common.merge_recursive(defaults, params)
# convert refs and boolean strings
for param_key, param_value in dict(params).items():
if param_value is not None:
param_value = params[param_key] = resolve_refs_recursively(stack_name, param_value, resources)
# Convert to boolean (TODO: do this recursively?)
if str(param_value).lower() in ['true', 'false']:
params[param_key] = str(param_value).lower() == 'true'
# convert any moto account IDs (123456789012) in ARNs to our format (000000000000)
params = fix_account_id_in_arns(params)
# convert data types (e.g., boolean strings to bool)
params = convert_data_types(func_details, params)
# remove None values, as they usually raise boto3 errors
params = remove_none_values(params)
# invoke function
try:
LOG.debug('Request for resource type "%s" in region %s: %s %s' % (
resource_type, aws_stack.get_region(), func_details['function'], params))
result = function(**params)
except Exception as e:
LOG.warning('Error calling %s with params: %s for resource: %s' % (function, params, resource))
raise e
# some resources have attached/nested resources which we need to create recursively now
if resource_type == 'ApiGateway::Method':
integration = resource_props.get('Integration')
if integration:
api_id = resolve_refs_recursively(stack_name, resource_props['RestApiId'], resources)
res_id = resolve_refs_recursively(stack_name, resource_props['ResourceId'], resources)
uri = integration.get('Uri')
if uri:
uri = resolve_refs_recursively(stack_name, uri, resources)
aws_stack.connect_to_service('apigateway').put_integration(
restApiId=api_id, resourceId=res_id,
httpMethod=resource_props['HttpMethod'], type=integration['Type'],
integrationHttpMethod=integration['IntegrationHttpMethod'], uri=uri)
elif resource_type == 'SNS::Topic':
subscriptions = resource_props.get('Subscription', [])
for subscription in subscriptions:
endpoint = resolve_refs_recursively(stack_name, subscription['Endpoint'], resources)
topic_arn = retrieve_topic_arn(params['Name'])
aws_stack.connect_to_service('sns').subscribe(
TopicArn=topic_arn, Protocol=subscription['Protocol'], Endpoint=endpoint)
elif resource_type == 'S3::Bucket':
tags = resource_props.get('Tags')
if tags:
aws_stack.connect_to_service('s3').put_bucket_tagging(
Bucket=params['Bucket'], Tagging={'TagSet': tags})
return result
def render_velocity_template(template, context, variables={}, as_json=False):
import airspeed
if not template:
return template
# Apply a few fixes below, to properly prepare the template...
# TODO: remove once this PR is merged: https://github.com/purcell/airspeed/pull/48
def expr_parse(self):
try:
self.identity_match(self.DOT)
self.expression = self.next_element(airspeed.VariableExpression)
except airspeed.NoMatch:
self.expression = self.next_element(airspeed.ArrayIndex)
self.subexpression = None
try:
self.subexpression = self.next_element(airspeed.SubExpression)
except airspeed.NoMatch:
pass
airspeed.SubExpression.parse = expr_parse
# TODO: remove once this PR is merged: https://github.com/purcell/airspeed/pull/48
def expr_calculate(self, current_object, loader, global_namespace):
args = [current_object, loader]
if not isinstance(self.expression, airspeed.ArrayIndex):
return self.expression.calculate(*(args + [global_namespace]))
index = self.expression.calculate(*args)
result = current_object[index]
if self.subexpression:
result = self.subexpression.calculate(result, loader,
global_namespace)
return result
airspeed.SubExpression.calculate = expr_calculate
# fix "#set" commands
template = re.sub(r"(^|\n)#\s+set(.*)", r"\1#set\2", template,
re.MULTILINE)
# enable syntax like "test#${foo.bar}"
empty_placeholder = " __pLaCe-HoLdEr__ "
template = re.sub(
r"([^\s]+)#\$({)?(.*)",
r"\1#%s$\2\3" % empty_placeholder,
template,
re.MULTILINE,
)
# add extensions for common string functions below
class ExtendedString(str):
def trim(self, *args, **kwargs):
return ExtendedString(self.strip(*args, **kwargs))
def toLowerCase(self, *args, **kwargs):
return ExtendedString(self.lower(*args, **kwargs))
def toUpperCase(self, *args, **kwargs):
return ExtendedString(self.upper(*args, **kwargs))
def apply(obj, **kwargs):
if isinstance(obj, dict):
for k, v in obj.items():
if isinstance(v, str):
obj[k] = ExtendedString(v)
return obj
# loop through the variables and enable certain additional util functions (e.g., string utils)
variables = variables or {}
recurse_object(variables, apply)
# prepare and render template
context_var = variables.get("context") or {}
context_var.setdefault("requestId", short_uid())
t = airspeed.Template(template)
var_map = {
"input": VelocityInput(context),
"util": VelocityUtil(),
"context": context_var,
}
var_map.update(variables or {})
replaced = t.merge(var_map)
# revert temporary changes from the fixes above
replaced = replaced.replace(empty_placeholder, "")
if as_json:
replaced = json.loads(replaced)
return replaced