def get_yaml_or_json_file(cls, url, working_dir):
"""
Load yaml or json from filesystem or s3
:param url: str
:param working_dir: str
:return: dict
"""
file_content = cls.get_file(url, working_dir)
try:
if url.lower().endswith(".json"):
return json.loads(file_content)
elif url.lower().endswith(".template"):
# try to load json
try:
return json.loads(file_content)
except Exception as e:
# try to load yaml
yaml.add_multi_constructor(u"",
cls.handle_yaml_constructors)
return yaml.load(file_content)
elif url.lower().endswith(".yml") or url.lower().endswith(".yaml"):
yaml.add_multi_constructor(u"", cls.handle_yaml_constructors)
return yaml.load(file_content)
else:
raise CfnSphereException(
"Invalid suffix, use [json|template|yml|yaml]")
except Exception as e:
raise CfnSphereException(e)
def get_yaml_or_json_file(cls, url, working_dir):
"""
Load yaml or json from filesystem or s3
:param url: str
:param working_dir: str
:return: dict
"""
file_content = cls.get_file(url, working_dir)
try:
if url.lower().endswith(".json"):
return json.loads(file_content, encoding='utf-8')
elif url.lower().endswith(".template"):
return json.loads(file_content, encoding='utf-8')
elif url.lower().endswith(".yml") or url.lower().endswith(".yaml"):
if hasattr(yaml, 'FullLoader'):
loader = yaml.FullLoader
else:
loader = yaml.Loader
yaml.add_multi_constructor(u"",
cls.handle_yaml_constructors,
Loader=loader)
return yaml.load(file_content, Loader=loader)
else:
raise CfnSphereException(
"Invalid suffix, use [json|template|yml|yaml]")
except Exception as e:
raise CfnSphereException(e)
def handle_yaml_constructors(loader, suffix, node):
"""
Constructor method for PyYaml to handle cfn intrinsic functions specified as yaml tags
"""
function_mapping = {
"!base64": ("Fn::Base64", lambda x: x),
"!and": ("Fn::And", lambda x: x),
"!equals": ("Fn::Equals", lambda x: x),
"!if": ("Fn::If", lambda x: x),
"!not": ("Fn::Not", lambda x: x),
"!or": ("Fn::Or", lambda x: x),
"!findinmap": ("Fn::FindInMap", lambda x: x),
"!getatt": ("Fn::GetAtt", lambda x: str(x).split(".",1)),
"!getazs": ("Fn::GetAZs", lambda x: x),
"!importvalue": ("Fn::ImportValue", lambda x: x),
"!join": ("Fn::Join", lambda x: [x[0], x[1]]),
"!select": ("Fn::Select", lambda x: x),
"!sub": ("Fn::Sub", lambda x: x),
"!ref": ("Ref", lambda x: x)
}
try:
function, value_transformer = function_mapping[str(suffix).lower()]
except KeyError as key:
raise CfnSphereException("Unsupported cfn intrinsic function tag found: {0}".format(key))
if isinstance(node, yaml.ScalarNode):
value = loader.construct_scalar(node)
elif isinstance(node, yaml.SequenceNode):
value = loader.construct_sequence(node)
elif isinstance(node, yaml.MappingNode):
value = loader.construct_mapping(node)
else:
raise CfnSphereException("Invalid yaml node found while handling cfn intrinsic function tags")
return {function: value_transformer(value)}
def extract_topic_arn(resource_description, parameters):
try:
parameter_name = resource_description['Properties']['TopicArn']
if isinstance(parameter_name, dict):
parameter_name = resource_description['Properties']['TopicArn']['Ref']
for key in parameters.keys():
if key == parameter_name:
return parameters[key]
raise CfnSphereException("Could not find a parameter value for {0}".format(parameter_name))
else:
return parameter_name
except KeyError:
raise CfnSphereException("Could not find TopicArn in Custom::SNS::Subscription properties")
def _parse_cli_parameters(parameters):
"""
Parse clix parameter tuple
:param parameters: tuple with n elements where n is number of cli parameters
:return: dict of stacks with k-v parameters
"""
param_dict = defaultdict(dict)
if parameters:
try:
for key_value_parameter_pair in parameters:
stack_and_parameter_key, parameter_value = key_value_parameter_pair.split(
"=", 1)
stack, parameter_key = stack_and_parameter_key.split(
".", 1)
stack_parameter = {
parameter_key.strip(): parameter_value.strip()
}
param_dict[stack.strip()].update(stack_parameter)
except (KeyError, ValueError):
raise CfnSphereException(
"""Format of input parameters is faulty.
Use 'stack1.param=value,stack2.param=value'""")
return param_dict
def get_images(self, name_pattern):
"""
Return list of AMIs matching the given name_pattern
:param name_pattern: str: AMI name pattern
:return: list(dict)
:raise CfnSphereException:
"""
filters = [{
'Name': 'name',
'Values': [name_pattern]
}, {
'Name': 'is-public',
'Values': ['false']
}, {
'Name': 'state',
'Values': ['available']
}, {
'Name': 'root-device-type',
'Values': ['ebs']
}]
try:
response = self.client.describe_images(ExecutableUsers=["self"],
Filters=filters)
except (BotoCoreError, ClientError) as e:
raise CfnSphereBotoError(e)
if not response['Images']:
raise CfnSphereException(
"Could not find any private and available Taupage AMI")
return response['Images']
def _is_valid_yaml(content):
try:
yaml.safe_load(content)
except yaml.YAMLError as e:
print(content)
raise CfnSphereException(
"Rendered file does not make valid yaml: {0}".format(e))
def handle_ssm_value(self, value):
parts = value.split('|')
if len(parts) == 3:
return str(self.ssm.get_parameter(parts[2]))
raise CfnSphereException(
"Invalid format for |ssm| macro, it must be |ssm|/path/to/parameter"
)
def get_cfn_api_server_time():
url = "https://aws.amazon.com"
try:
header_date = urllib2.urlopen(url).info().get('Date')
return parser.parse(header_date)
except Exception as e:
raise CfnSphereException("Could not get AWS server time from {0}. Error: {1}".format(url, e))
def handle_file_value(value, working_dir):
components = value.split('|', 3)
if len(components) == 3:
url = components[2]
return FileLoader.get_file(url, working_dir)
elif len(components) == 4:
url = components[2]
pattern = components[3]
file_content = FileLoader.get_yaml_or_json_file(url, working_dir)
try:
return jmespath.search(pattern, file_content)
except JMESPathError as e:
raise CfnSphereException(e)
else:
raise CfnSphereException(
"Invalid format for |File| macro, it must be |File|<path>[|<pattern>]"
)
def _s3_get_file(url):
"""
Load file from s3
:param url: str
:return: str(utf-8)
"""
try:
return S3().get_contents_from_url(url)
except Exception as e:
raise CfnSphereException("Could not load file from {0}: {1}".format(url, e))
def get_cfn_api_server_time():
url = "http://aws.amazon.com"
try:
header_date = urllib2.urlopen(url).info().get('Date')
return datetime.datetime.strptime(header_date,
'%a, %d %b %Y %H:%M:%S GMT')
except Exception as e:
raise CfnSphereException(
"Could not get AWS server time from {0}. Error: {1}".format(
url, e))
def _https_get_file(url):
"""
Load file from https
:param url: str
:return: str(utf-8)
"""
try:
with urllib.request.urlopen(url) as response:
return BeautifulSoup(response.read(), 'html.parser').prettify()
except Exception as e:
raise CfnSphereException(
"Could not load file from {0}: {1}".format(url, e))
def get_output_value(self, key):
"""
Get value for a specific output key in format <stack-name>.<output>.
:param key: str <stack-name>.<output>
:return: str
"""
artifacts = self.get_stack_outputs()
self.logger.debug("Looking up key: {0}".format(key))
self.logger.debug("Found artifacts: {0}".format(artifacts))
try:
artifact = artifacts[key]
return artifact
except KeyError:
raise CfnSphereException("Could not get a valid value for {0}.".format(key))
def get_latest_value(self, key, value, stack_name):
try:
if self.cfn.stack_exists(stack_name):
latest_stack_parameters = self.cfn.get_stack_parameters_dict(stack_name)
latest_value = latest_stack_parameters.get(key, None)
if latest_value:
self.logger.info("Will keep '{0}' as latest value for {1}".format(latest_value, key))
return latest_value
else:
return self.get_default_from_keep_value(value)
else:
return self.get_default_from_keep_value(value)
except CfnSphereBotoError as e:
raise CfnSphereException("Could not get latest value for {0}: {1}".format(key, e))
def handle_kms_value(self, value):
parts = value.split('|')
if len(parts) == 3:
return str(self.kms.decrypt(parts[2]))
elif len(parts) == 4:
return str(
self.kms.decrypt(parts[3],
encryption_context=kv_list_string_to_dict(
parts[2])))
else:
raise CfnSphereException(
"Invalid format for |Kms| macro, it must be |Kms[|<encryption_context>]|<ciphertext>"
)
def parse_stack_reference_value(value):
if not value:
return None, None
if value.lower().startswith('|ref|'):
components = value.split('|')
if len(components) != 3:
raise CfnSphereException(
"Stack output reference must be like '|ref|stack.output'")
reference = components[2]
reference_components = reference.split('.')
if len(reference_components) != 2:
raise CfnSphereException(
"Stack output reference must be like '|ref|stack.output'")
stack_name = reference_components[0]
output_name = reference_components[1]
return stack_name, output_name
else:
return None, None
def _fs_get_file(url, working_dir):
"""
Load file from filesystem
:param url: str template path
:return: str(utf-8)
"""
if not os.path.isabs(url) and working_dir:
url = os.path.join(working_dir, url)
try:
with codecs.open(url, 'r', encoding='utf-8') as f:
return f.read()
except Exception as e:
raise CfnSphereException("Could not load file from {0}: {1}".format(url, e))
def decrypt(self, encrypted_value, encryption_context=None):
if encryption_context is None:
encryption_context = {}
try:
ciphertext_blob = base64.b64decode(encrypted_value.encode())
response = self.client.decrypt(
CiphertextBlob=ciphertext_blob,
EncryptionContext=encryption_context)
return response['Plaintext'].decode('utf-8')
except Boto3Error as e:
raise CfnSphereBotoError(e)
except ClientError as e:
raise CfnSphereException(e)
def kv_list_to_dict(items):
"""
Converts a list of strings with k=v to dict {k:v}
:param items: list(string)
:return: dict
"""
result = {}
for item in items:
parts = str(item).split("=")
if not len(parts) == 2:
raise CfnSphereException("Could not parse kv pair: {0}, please ensure it is passed as k=v".format(items))
result[parts[0]] = parts[1]
return result
def _write_file(file_path, working_dir, content):
if os.path.isabs(file_path):
path = file_path
else:
path = os.path.join(working_dir, file_path)
try:
target_dir = os.path.dirname(path)
if not os.path.exists(target_dir):
os.mkdir(target_dir)
with open(path, "w") as f:
f.write(content)
except Exception as e:
raise CfnSphereException(e)
def get_output_value(self, stack_outputs, stack, output_key):
"""
Get value for a specific output key in format <stack-name>.<output>.
:param output_key: str <stack-name>.<output>
:return: str
"""
self.logger.debug("Looking up output {0} from stack {1}".format(
output_key, stack))
try:
artifact = stack_outputs[stack][output_key]
return artifact
except KeyError:
raise CfnSphereException(
"Could not get a valid value for {0}.".format(output_key))
def resolve_parameter_value(self, key, value, stack_name, stack_config,
stack_outputs):
if isinstance(value, list):
self.logger.debug("List parameter found for {0}".format(key))
for i, item in enumerate(value):
value[i] = self.resolve_parameter_value(
key, item, stack_name, stack_config, stack_outputs)
return self.convert_list_to_string(value)
elif isinstance(value, string_types):
if DependencyResolver.is_parameter_reference(value):
referenced_stack, output_name = DependencyResolver.parse_stack_reference_value(
value)
return str(
self.get_output_value(stack_outputs, referenced_stack,
output_name))
elif self.is_keep_value(value):
return str(self.get_latest_value(key, value, stack_name))
elif self.is_taupage_ami_reference(value):
return str(self.ec2.get_latest_taupage_image_id())
elif self.is_kms(value):
return self.handle_kms_value(value)
elif self.is_ssm(value):
return self.handle_ssm_value(value)
elif self.is_file(value):
return self.handle_file_value(value, stack_config.working_dir)
else:
return value
elif isinstance(value, bool):
return str(value).lower()
elif isinstance(value, (int, float)):
return str(value)
elif not value:
raise CfnSphereException(
"Parameter {0} does not seem to have a value".format(key))
else:
raise NotImplementedError(
"Cannot handle {0} type for key: {1}".format(type(value), key))
def get_taupage_images(self):
filters = {
'name': ['Taupage-AMI-*'],
'is-public': ['false'],
'state': ['available'],
'root-device-type': ['ebs']
}
try:
response = self.conn.get_all_images(executable_by=["self"],
filters=filters)
except BotoServerError as e:
raise CfnSphereBotoError(e)
if not response:
raise CfnSphereException(
"Could not find any private and available Taupage AMI")
return response
def handle_sns_subscription(cls, resource_description, stack):
logger = get_logger()
queue_prefix = stack.name + '-' + resource_description['Properties']['QueueResourceName']
topic_region = resource_description['Properties'].get('TopicRegion', stack.region)
topic_arn = cls.extract_topic_arn(resource_description, stack.parameters)
sqs_conn = sqs.connect_to_region(stack.region)
sns_conn = sns.connect_to_region(topic_region)
queues = sqs_conn.get_all_queues(prefix=queue_prefix)
if len(queues) != 1:
raise CfnSphereException(
"Found {0} queues matching the prefix: {1}. Should be 1.".format(len(queues), queue_prefix))
queue = queues[0]
logger.info("Subscribing queue {0} to topic {1} in {2}".format(queue.name, topic_arn, topic_region))
sns_conn.subscribe_sqs_queue(topic_arn, queue)
def _is_valid_json(content):
try:
json.loads(content)
except ValueError as e:
raise CfnSphereException(
"Rendered file does not make valid json: {0}".format(e))
def handle_sns_subscription(cls, resource_description, stack):
raise CfnSphereException(
"The Cfn-Sphere PostCustomResource feature has been removed")
