def test_getatt_from_yaml():
"""
Test that we correctly convert the short form of GetAtt
into the correct JSON format from YAML
"""
source = """
- !GetAtt foo.bar
- Fn::GetAtt: [foo, bar]
"""
expected = [
{
"Fn::GetAtt": ["foo", "bar"]
},
{
"Fn::GetAtt": ["foo", "bar"]
},
]
# No clean
actual = cfn_flip.to_json(source, clean_up=False)
assert load_json(actual) == expected
# With clean
actual = cfn_flip.to_json(source, clean_up=True)
assert load_json(actual) == expected
def test_flip_to_yaml(input_json, input_yaml, parsed_yaml):
"""
Test that flip performs correctly transforming from json to yaml
"""
actual = cfn_flip.flip(input_json)
assert actual == input_yaml + "\n"
# The result should not parse as json
with pytest.raises(ValueError):
load_json(actual)
parsed_actual = load_yaml(actual)
assert parsed_actual == parsed_yaml
def test_to_yaml_with_json(input_json, parsed_yaml):
"""
Test that to_yaml performs correctly
"""
actual = cfn_flip.to_yaml(input_json)
# The result should not parse as json
with pytest.raises(ValueError):
load_json(actual)
parsed_actual = load_yaml(actual)
assert parsed_actual == parsed_yaml
def test_flip_to_clean_yaml(input_json, clean_yaml, parsed_clean_yaml):
"""
Test that flip performs correctly transforming from json to yaml
and the `clean_up` flag is active
"""
actual = cfn_flip.flip(input_json, clean_up=True)
assert actual == clean_yaml + "\n"
# The result should not parse as json
with pytest.raises(ValueError):
load_json(actual)
parsed_actual = load_yaml(actual)
assert parsed_actual == parsed_clean_yaml
def test_dump_json():
"""
JSON dumping just needs to know about datetimes,
provide a nice indent, and preserve order
"""
import sys
source = ODict((
("z", datetime.time(3, 45)),
("m", datetime.date(2012, 5, 2)),
("a", datetime.datetime(2012, 5, 2, 3, 45)),
))
actual = dump_json(source)
assert load_json(actual) == {
"z": "03:45:00",
"m": "2012-05-02",
"a": "2012-05-02T03:45:00",
}
if sys.version_info < (3, 6):
fail_message = r"\(1\+1j\) is not JSON serializable"
elif sys.version_info < (3, 7):
fail_message = "Object of type 'complex' is not JSON serializable"
else:
fail_message = "Object of type complex is not JSON serializable"
with pytest.raises(TypeError, match=fail_message):
dump_json({
"c": 1 + 1j,
})
def test_to_yaml_with_long_json(input_long_json):
"""
Test that to_yaml performs correctly
"""
actual = cfn_flip.to_yaml(input_long_json)
# The result should not parse as json
with pytest.raises(ValueError):
load_json(actual)
parsed_actual = load_yaml(actual)
assert parsed_actual['TooShort'] == "foo\nbar\nbaz\nquuux"
assert 'WideText: >-' in actual
assert 'TooShort: "foo' in actual
def test_to_json_with_yaml(input_yaml, parsed_json):
"""
Test that to_json performs correctly
"""
actual = cfn_flip.to_json(input_yaml)
assert load_json(actual) == parsed_json
def test_flip_to_multibyte_yaml(multibyte_yaml, parsed_multibyte_json):
"""
Test that load multibyte file performs correctly
"""
actual = cfn_flip.to_json(multibyte_yaml)
assert load_json(actual) == parsed_multibyte_json
def test_flip_to_json_with_condition():
"""
Test that the Condition key is correctly converted
"""
source = """
MyAndCondition: !And
- !Equals ["sg-mysggroup", !Ref "ASecurityGroup"]
- !Condition SomeOtherCondition
"""
expected = {
"MyAndCondition": {
"Fn::And": [{
"Fn::Equals": ["sg-mysggroup", {
"Ref": "ASecurityGroup"
}]
}, {
"Condition": "SomeOtherCondition"
}]
}
}
actual = cfn_flip.to_json(source, clean_up=True)
assert load_json(actual) == expected
def test_flip_to_json(input_yaml, input_json, parsed_json):
"""
Test that flip performs correctly transforming from yaml to json
"""
actual = cfn_flip.flip(input_yaml)
assert load_json(actual) == parsed_json
def test_no_flip_with_json(input_json, parsed_json):
"""
We should be able to submit JSON and get JSON back
"""
actual = cfn_flip.flip(input_json, no_flip=True)
assert load_json(actual) == parsed_json
def test_flip_with_json_output(input_yaml, parsed_json):
"""
We should be able to specify that the output is JSON
"""
actual = cfn_flip.flip(input_yaml, out_format="json")
assert load_json(actual) == parsed_json
def test_no_flip_with_explicit_json(input_json, parsed_json):
"""
We should be able to submit JSON and get JSON back
and specify the output format explicity
"""
actual = cfn_flip.flip(input_json, out_format="json", no_flip=True)
assert load_json(actual) == parsed_json
def test_flip_to_clean_json(input_yaml, clean_json, parsed_clean_json):
"""
Test that flip performs correctly transforming from yaml to json
and the `clean_up` flag is active
"""
actual = cfn_flip.flip(input_yaml, clean_up=True)
assert load_json(actual) == parsed_clean_json
def test_to_json_with_json(input_json, parsed_json):
"""
Test that to_json still works when passed json
(All json is valid yaml)
"""
actual = cfn_flip.to_json(input_json)
assert load_json(actual) == parsed_json
def to_yaml(template, clean_up=False, long_form=False):
"""
Assume the input is JSON and convert to YAML
"""
data = load_json(template)
if clean_up:
data = clean(data)
return dump_yaml(data, clean_up, long_form)
def load(template):
"""
Try to guess the input format
"""
try:
data = load_json(template)
return data, "json"
except ValueError:
data = load_yaml(template)
return data, "yaml"
def test_flip_to_json_with_multi_level_getatt():
"""
Test that we correctly convert multi-level Fn::GetAtt
from YAML to JSON format
"""
data = "!GetAtt 'First.Second.Third'\n"
expected = {"Fn::GetAtt": ["First", "Second.Third"]}
actual = cfn_flip.to_json(data, clean_up=True)
assert load_json(actual) == expected
def yaml_to_json(p_input_file):
json_out = ROOT_DIR+'/../../../tacocat_cf.json'
with open( p_input_file,'r') as yaml_in:
yaml_data = load_yaml(yaml_in)
yaml_data = clean(yaml_data)
with open(json_out,'w') as out_file:
json_data = dump_json(yaml_data)
json_data = load_json(json_data)
json.dump(json_data ,out_file, sort_keys=True, indent=4, separators=(',', ': '))
print('*--*--*--*--*--*--*--*--*--*--*--*--**--*--*--*--*--*--*--*--*--*--*--*')
print('The new JSON file has been created in the root folder of the project with the name tacocat_cf.json')
print('*--*--*--*--*--*--*--*--*--*--*--*--**--*--*--*--*--*--*--*--*--*--*--*')
def flip(template,
in_format=None,
out_format=None,
clean_up=False,
no_flip=False,
long_form=False):
"""
Figure out the input format and convert the data to the opposing output format
"""
# Do we need to figure out the input format?
if not in_format:
# Load the template as JSON?
if (out_format == "json" and no_flip) or (out_format == "yaml"
and not no_flip):
in_format = "json"
elif (out_format == "yaml" and no_flip) or (out_format == "json"
and not no_flip):
in_format = "yaml"
# Load the data
if in_format == "json":
data = load_json(template)
elif in_format == "yaml":
data = load_yaml(template)
else:
data, in_format = load(template)
# Clean up?
if clean_up:
data = clean(data)
# Figure out the output format
if not out_format:
if (in_format == "json" and no_flip) or (in_format == "yaml"
and not no_flip):
out_format = "json"
else:
out_format = "yaml"
# Finished!
if out_format == "json":
if sys.version[0] == "3":
return dump_json(data)
else:
return dump_json(data).encode('utf-8')
return dump_yaml(data, clean_up, long_form)
def test_flip_to_json_with_datetimes():
"""
Test that the json encoder correctly handles dates and datetimes
"""
tricky_data = """
a date: 2017-03-02
a datetime: 2017-03-02 19:52:00
"""
actual = cfn_flip.to_json(tricky_data)
parsed_actual = load_json(actual)
assert parsed_actual == {
"a date": "2017-03-02",
"a datetime": "2017-03-02T19:52:00",
}
def convert_yaml_to_json(yaml_file):
converted_file = ROOT_DIR + '/../../../tacocat_cf.json'
with open(yaml_file, 'r') as file_in:
yaml_data = load_yaml(file_in)
yaml_data = clean(yaml_data)
with open(converted_file, 'w') as file_out:
json_data = dump_json(yaml_data)
json_data = load_json(json_data)
json.dump(json_data,
file_out,
sort_keys=True,
indent=4,
separators=(',', ': '))
print(
'#####--Converted successfully, please check the root folder with the name tacocat_cf.json--#####'
)
def test_load_json():
"""
Should map to an ordered dict
"""
source = """
{
"z": "first",
"m": "middle",
"a": "last"
}
"""
actual = load_json(source)
assert type(actual) == ODict
assert list(actual.keys()) == ["z", "m", "a"]
assert actual["z"] == "first"
assert actual["m"] == "middle"
assert actual["a"] == "last"
def flip(template, out_format=None, clean_up=False, no_flip=False, long_form=False):
"""
Figure out the input format and convert the data to the opposing output format
"""
data = None
in_format = None
if no_flip:
in_format = out_format
elif out_format == "json":
in_format = "yaml"
elif out_format == "yaml":
in_format = "json"
if in_format == "json":
data = load_json(template)
elif in_format == "yaml":
data = load_yaml(template)
else:
try:
data, in_format = load(template)
except Exception:
raise Exception("Could not determine the input format")
if no_flip:
out_format = in_format
elif in_format == "json":
out_format = "yaml"
else:
out_format = "json"
if clean_up:
data = clean(data)
if out_format == "json":
return dump_json(data)
return dump_yaml(data, clean_up, long_form)
def test_dump_json():
"""
JSON dumping just needs to know about datetimes,
provide a nice indent, and preserve order
"""
source = ODict((
("z", datetime.time(3, 45)),
("m", datetime.date(2012, 5, 2)),
("a", datetime.datetime(2012, 5, 2, 3, 45)),
))
actual = dump_json(source)
assert load_json(actual) == {
"z": "03:45:00",
"m": "2012-05-02",
"a": "2012-05-02T03:45:00",
}
with pytest.raises(TypeError, message="complex is not JSON serializable"):
dump_json({
"c": 1 + 1j,
})
def run(self):
stack = self.ensure_stack_is_in_complete_status()
status = stack.get("StackStatus")
with self.spoke_regional_client("cloudformation") as cloudformation:
if status == "ROLLBACK_COMPLETE":
if self.should_delete_rollback_complete_stacks:
cloudformation.ensure_deleted(
StackName=self.stack_name_to_use)
else:
raise Exception(
f"Stack: {self.stack_name_to_use} is in ROLLBACK_COMPLETE and need remediation"
)
task_output = dict(
**self.params_for_results_display(),
account_parameters=tasks.unwrap(self.account_parameters),
launch_parameters=tasks.unwrap(self.launch_parameters),
manifest_parameters=tasks.unwrap(self.manifest_parameters),
)
all_params = self.get_parameter_values()
template_to_provision_source = self.input().get("template").open(
"r").read()
try:
template_to_provision = cfn_tools.load_yaml(
template_to_provision_source)
except Exception:
try:
template_to_provision = cfn_tools.load_json(
template_to_provision_source)
except Exception:
raise Exception("Could not parse new template as YAML or JSON")
params_to_use = dict()
for param_name, p in template_to_provision.get("Parameters",
{}).items():
if all_params.get(param_name, p.get("DefaultValue")) is not None:
params_to_use[param_name] = all_params.get(
param_name, p.get("DefaultValue"))
existing_stack_params_dict = dict()
existing_template = ""
if status in [
"CREATE_COMPLETE",
"UPDATE_ROLLBACK_COMPLETE",
"UPDATE_COMPLETE",
"IMPORT_COMPLETE",
"IMPORT_ROLLBACK_COMPLETE",
]:
with self.spoke_regional_client(
"cloudformation") as cloudformation:
existing_stack_params_dict = {}
summary_response = cloudformation.get_template_summary(
StackName=self.stack_name_to_use, )
for parameter in summary_response.get("Parameters"):
existing_stack_params_dict[parameter.get(
"ParameterKey")] = parameter.get("DefaultValue")
for stack_param in stack.get("Parameters", []):
existing_stack_params_dict[stack_param.get(
"ParameterKey")] = stack_param.get("ParameterValue")
template_body = cloudformation.get_template(
StackName=self.stack_name_to_use,
TemplateStage="Original").get("TemplateBody")
try:
existing_template = cfn_tools.load_yaml(template_body)
except Exception:
try:
existing_template = cfn_tools.load_json(template_body)
except Exception:
raise Exception(
"Could not parse existing template as YAML or JSON"
)
template_to_use = cfn_tools.dump_yaml(template_to_provision)
if status == "UPDATE_ROLLBACK_COMPLETE":
need_to_provision = True
else:
if existing_stack_params_dict == params_to_use:
self.info(f"params unchanged")
if template_to_use == cfn_tools.dump_yaml(existing_template):
self.info(f"template the same")
need_to_provision = False
else:
self.info(f"template changed")
need_to_provision = True
else:
self.info(f"params changed")
need_to_provision = True
if need_to_provision:
provisioning_parameters = []
for p in params_to_use.keys():
provisioning_parameters.append({
"ParameterKey":
p,
"ParameterValue":
params_to_use.get(p)
})
with self.spoke_regional_client(
"cloudformation") as cloudformation:
a = dict(
StackName=self.stack_name_to_use,
TemplateBody=template_to_use,
ShouldUseChangeSets=False,
Capabilities=self.capabilities,
Parameters=provisioning_parameters,
ShouldDeleteRollbackComplete=self.
should_delete_rollback_complete_stacks,
)
if self.use_service_role:
a["RoleARN"] = config.get_puppet_stack_role_arn(
self.account_id)
cloudformation.create_or_update(**a)
task_output["provisioned"] = need_to_provision
self.info(f"self.execution is {self.execution}")
if self.execution == constants.EXECUTION_MODE_HUB:
self.info(
f"Running in execution mode: {self.execution}, checking for SSM outputs"
)
if len(self.ssm_param_outputs) > 0:
with self.spoke_regional_client(
"cloudformation") as spoke_cloudformation:
stack_details = aws.get_stack_output_for(
spoke_cloudformation,
self.stack_name_to_use,
)
for ssm_param_output in self.ssm_param_outputs:
self.info(
f"writing SSM Param: {ssm_param_output.get('stack_output')}"
)
with self.hub_client("ssm") as ssm:
found_match = False
# TODO push into another task
for output in stack_details.get("Outputs", []):
if output.get("OutputKey") == ssm_param_output.get(
"stack_output"):
ssm_parameter_name = ssm_param_output.get(
"param_name")
ssm_parameter_name = ssm_parameter_name.replace(
"${AWS::Region}", self.region)
ssm_parameter_name = ssm_parameter_name.replace(
"${AWS::AccountId}", self.account_id)
found_match = True
ssm.put_parameter_and_wait(
Name=ssm_parameter_name,
Value=output.get("OutputValue"),
Type=ssm_param_output.get(
"param_type", "String"),
Overwrite=True,
)
if not found_match:
raise Exception(
f"[{self.uid}] Could not find match for {ssm_param_output.get('stack_output')}"
)
self.write_output(task_output)
else:
self.write_output(task_output)
def parsed_multibyte_json():
return load_json(multibyte_json())
def parsed_clean_json():
return load_json(clean_json())
def parsed_json():
return load_json(input_json())
def run(self):
status = self.get_current_status()
if status == "-":
self.write_result(
"-",
self.version_id,
effect=constants.CHANGE,
current_status="-",
active="N/A",
notes="Stack would be created",
)
elif status == "ROLLBACK_COMPLETE":
if self.should_delete_rollback_complete_stacks:
self.write_result(
"-",
self.version_id,
effect=constants.CHANGE,
current_status="-",
active="N/A",
notes="Stack would be replaced",
)
else:
self.write_result(
"-",
"-",
effect=constants.NO_CHANGE,
current_status="-",
active="N/A",
notes="Stack needs remediation - it's in ROLLBACK_COMPLETE",
)
else:
task_output = dict(
**self.params_for_results_display(),
account_parameters=tasks.unwrap(self.account_parameters),
launch_parameters=tasks.unwrap(self.launch_parameters),
manifest_parameters=tasks.unwrap(self.manifest_parameters),
)
all_params = self.get_parameter_values()
template_to_provision_source = self.input().get("template").open(
"r").read()
try:
template_to_provision = cfn_tools.load_yaml(
template_to_provision_source)
except Exception:
try:
template_to_provision = cfn_tools.load_json(
template_to_provision_source)
except Exception:
raise Exception(
"Could not parse new template as YAML or JSON")
params_to_use = dict()
for param_name, p in template_to_provision.get("Parameters",
{}).items():
if all_params.get(param_name,
p.get("DefaultValue")) is not None:
params_to_use[param_name] = all_params.get(
param_name, p.get("DefaultValue"))
existing_stack_params_dict = dict()
existing_template = ""
if status in [
"CREATE_COMPLETE",
"UPDATE_ROLLBACK_COMPLETE",
"UPDATE_COMPLETE",
"IMPORT_COMPLETE",
"IMPORT_ROLLBACK_COMPLETE",
]:
with self.spoke_regional_client(
"cloudformation") as cloudformation:
existing_stack_params_dict = {}
stack = cloudformation.describe_stacks(
StackName=self.stack_name).get("Stacks")[0]
summary_response = cloudformation.get_template_summary(
StackName=self.stack_name, )
for parameter in summary_response.get("Parameters"):
existing_stack_params_dict[parameter.get(
"ParameterKey")] = parameter.get("DefaultValue")
for stack_param in stack.get("Parameters", []):
existing_stack_params_dict[stack_param.get(
"ParameterKey")] = stack_param.get(
"ParameterValue")
template_body = cloudformation.get_template(
StackName=self.stack_name,
TemplateStage="Original").get("TemplateBody")
try:
existing_template = cfn_tools.load_yaml(template_body)
except Exception:
try:
existing_template = cfn_tools.load_json(
template_body)
except Exception:
raise Exception(
"Could not parse existing template as YAML or JSON"
)
template_to_use = cfn_tools.dump_yaml(template_to_provision)
if status == "UPDATE_ROLLBACK_COMPLETE":
self.write_result(
"?",
self.version_id,
effect=constants.CHANGE,
current_status=status,
active="N/A",
notes="Stack would be updated",
)
else:
if existing_stack_params_dict == params_to_use:
self.info(f"params unchanged")
if template_to_use == cfn_tools.dump_yaml(
existing_template):
self.info(f"template the same")
self.write_result(
"?",
self.version_id,
effect=constants.NO_CHANGE,
current_status=status,
active="N/A",
notes="No change",
)
else:
self.info(f"template changed")
self.write_result(
"?",
self.version_id,
effect=constants.CHANGE,
current_status=status,
active="N/A",
notes="Template has changed",
)
else:
self.info(f"params changed")
self.write_result(
"?",
self.version_id,
effect=constants.CHANGE,
current_status=status,
active="N/A",
notes="Parameters have changed",
)