def k8s_cfn_repo(ctx, **_):
# type: (click.Context, Any) -> None
"""Generate a sample CloudFormation project using Kubernetes."""
src = TEMPLATES / "k8s-cfn-repo"
dest = Path.cwd() / "k8s-cfn-infrastructure"
copy_sample(ctx, src, dest)
convert_gitignore(dest / "_gitignore")
master_templates = dest / "k8s-master.cfn/templates"
worker_templates = dest / "k8s-workers.cfn/templates"
env = {"namespace": "test"}
LOGGER.verbose("rendering master templates...")
master_templates.mkdir()
(master_templates / "k8s_iam.yaml").write_text(
six.u(to_yaml(Iam("test", CFNginContext(env.copy()), None).to_json())))
(master_templates / "k8s_master.yaml").write_text(
six.u(
to_yaml(
Cluster("test", CFNginContext(env.copy()), None).to_json())))
LOGGER.verbose("rendering worker templates...")
worker_templates.mkdir()
(worker_templates / "k8s_workers.yaml").write_text(
six.u(
to_yaml(
NodeGroup("test", CFNginContext(env.copy()), None).to_json())))
LOGGER.success("Sample k8s infrastructure repo created at %s", dest)
LOGGER.notice("See the README for setup and deployment instructions.")
def get_json_or_yaml(data, output_format='json'):
'''
Convert into json or yaml from
json or yaml.
'''
# Make all dicts a string first
if isinstance(data, dict):
data = json.dumps(data)
# Get JSON, no matter what input (yaml or json)
try:
data = to_json(data)
except ValueError:
data = to_yaml(data)
data = to_json(data)
# Create python dict
data = json.loads(data)
# Sort python dict
data = SortedDict(**data)
# Quote keys
data = quote_json(data)
# Return json or yaml
if output_format == 'yaml':
data = to_yaml(json.dumps(data))
return data
def generate_service(self):
self._add_service_parameters()
self._add_service_outputs()
self._fetch_current_desired_count()
self._add_ecs_service_iam_role()
self._add_cluster_services()
key = uuid.uuid4().hex + '.yml'
if len(to_yaml(self.template.to_json())) > 51000:
try:
self.client.put_object(
Body=to_yaml(self.template.to_json()),
Bucket=self.bucket_name,
Key=key,
)
template_url = f'https://{self.bucket_name}.s3.amazonaws.com/{key}'
return template_url, 'TemplateURL', key
except ClientError as boto_client_error:
error_code = boto_client_error.response['Error']['Code']
if error_code == 'AccessDenied':
raise UnrecoverableException(
f'Unable to store cloudlift service template in S3 bucket at {self.bucket_name}'
)
else:
raise boto_client_error
else:
return to_yaml(self.template.to_json()), 'TemplateBody', ''
def test_to_yaml_with_yaml(input_yaml):
"""
Test that to_yaml fails with a ValueError when passed yaml
Yaml is not valid json
"""
with pytest.raises(Exception, message="Invalid JSON"):
cfn_flip.to_yaml(input_yaml)
def test_to_yaml_with_yaml(fail_message, input_yaml):
"""
Test that to_yaml fails with a ValueError when passed yaml
Yaml is not valid json
"""
with pytest.raises(JSONDecodeError, match=fail_message):
cfn_flip.to_yaml(input_yaml)
def test_flip_to_yaml_with_clean_getatt():
"""
The clean flag should convert Fn::GetAtt to its short form
"""
data = """
{
"Fn::GetAtt": ["Left", "Right"]
}
"""
expected = "!GetAtt 'Left.Right'\n"
assert cfn_flip.to_yaml(data, clean_up=False) == expected
assert cfn_flip.to_yaml(data, clean_up=True) == expected
def generate_sample_k8s_cfn_repo(env_root):
"""Generate sample k8s infrastructure repo."""
repo_dir = os.path.join(env_root, 'k8s-cfn-infrastructure')
if os.path.isdir(repo_dir):
LOGGER.error("Error generating sample repo -- directory %s "
"already exists!",
repo_dir)
sys.exit(1)
from runway.blueprints.k8s.k8s_master import Cluster
from runway.blueprints.k8s.k8s_iam import Iam
from runway.blueprints.k8s.k8s_workers import NodeGroup as WorkerNodeGroup
shutil.copytree(
os.path.join(ROOT,
'templates',
'k8s-cfn-repo'),
repo_dir
)
os.rename(os.path.join(repo_dir, '_gitignore'),
os.path.join(repo_dir, '.gitignore'))
# Generate masters CFN templates from blueprints
master_template_dir = os.path.join(repo_dir, 'k8s-master.cfn', 'templates')
os.mkdir(master_template_dir)
with open(os.path.join(master_template_dir, 'k8s_iam.yaml'), 'w') as stream:
stream.write(to_yaml(Iam('test',
Context({"namespace": "test"}),
None).to_json()))
with open(os.path.join(master_template_dir, 'k8s_master.yaml'), 'w') as stream:
stream.write(to_yaml(Cluster('test',
Context({"namespace": "test"}),
None).to_json()))
# Generate workers CFN template from blueprint
worker_template_dir = os.path.join(repo_dir,
'k8s-workers.cfn',
'templates')
os.mkdir(worker_template_dir)
with open(os.path.join(worker_template_dir,
'k8s_workers.yaml'), 'w') as stream:
stream.write(to_yaml(WorkerNodeGroup('test',
Context({"namespace": "test"}),
None).to_json()))
LOGGER.info("Sample k8s infrastructure repo created at %s",
repo_dir)
LOGGER.info('(see its README for setup and deployment instructions)')
def generate_service(self):
self._add_service_parameters()
self._add_service_outputs()
self._fetch_current_desired_count()
self._add_ecs_service_iam_role()
self._add_cluster_services()
return to_yaml(self.template.to_json())
def test_flip_to_yaml_with_json_literal(input_json_with_literal, parsed_yaml_with_json_literal):
"""
Test that load json with json payload that must stay json when converted to yaml
"""
actual = cfn_flip.to_yaml(input_json_with_literal)
assert load_yaml(actual) == parsed_yaml_with_json_literal
def test_flip_to_yaml_with_clean_getatt(self):
"""
The clean flag should convert Fn::GetAtt to its short form
"""
data = """
{
"Fn::GetAtt": ["Left", "Right"]
}
"""
expected = "!GetAtt 'Left.Right'\n"
self.assertEqual(cfn_flip.to_yaml(data, clean_up=False), expected)
self.assertEqual(cfn_flip.to_yaml(data, clean_up=True), expected)
def generate_tfstate_cfn_template():
"""Return rendered CFN template yaml."""
# pylint: disable=import-outside-toplevel
from runway.blueprints.tf_state import TfState
return to_yaml(
TfState('test', Context({"namespace": "test"}), None).to_json())
def test_clean_flip_to_yaml_with_newlines():
"""
Test that strings containing newlines use blockquotes when using "clean"
"""
source = dump_json(ODict((
("outer", ODict((
("inner", "#!/bin/bash\nyum -y update\nyum install python"),
("subbed", ODict((
("Fn::Sub", "The cake\nis\n${CakeType}"),
))),
))),
)))
expected = """outer:
inner: |-
#!/bin/bash
yum -y update
yum install python
subbed: !Sub |-
The cake
is
${CakeType}
"""
assert cfn_flip.to_yaml(source, clean_up=True) == expected
def test_flip_to_multibyte_json(multibyte_json, parsed_multibyte_yaml):
"""
Test that load multibyte file performs correctly
"""
actual = cfn_flip.to_yaml(multibyte_json)
assert load_yaml(actual) == parsed_multibyte_yaml
def test_quoted_digits():
"""
Any value that is composed entirely of digits
should be quoted for safety.
CloudFormation is happy for numbers to appear as strings.
But the opposite (e.g. account numbers as numbers) can cause issues
See https://github.com/awslabs/aws-cfn-template-flip/issues/41
"""
value = dump_json(ODict((
("int", 123456),
("float", 123.456),
("oct", "0123456"),
("bad-oct", "012345678"),
("safe-oct", "0o123456"),
("string", "abcdef"),
)))
expected = "\n".join((
"int: 123456",
"float: 123.456",
"oct: '0123456'",
"bad-oct: '012345678'",
"safe-oct: '0o123456'",
"string: abcdef",
""
))
actual = cfn_flip.to_yaml(value)
assert actual == expected
def to_yaml(self,
clean_up: bool = False,
long_form: bool = False,
sort_keys: bool = True) -> str:
return cfn_flip.to_yaml( # type: ignore
self.to_json(sort_keys=sort_keys),
clean_up=clean_up,
long_form=long_form)
def test_to_yaml_with_yaml(self):
"""
Test that to_yaml fails with a ValueError when passed yaml
Yaml is not valid json
"""
with self.assertRaises(ValueError):
actual = cfn_flip.to_yaml(self.input_yaml)
def test_to_yaml_with_yaml(input_yaml, parsed_yaml):
"""
Test that to_yaml still works when passed yaml
"""
actual = cfn_flip.to_yaml(input_yaml)
assert load_yaml(actual) == parsed_yaml
def test_to_yaml_with_yaml(self):
"""
Test that to_yaml fails with a ValueError when passed yaml
Yaml is not valid json
"""
with self.assertRaisesRegexp(Exception, "Invalid JSON"):
actual = cfn_flip.to_yaml(self.input_yaml)
def test_to_yaml_with_yaml(self):
"""
Test that to_yaml fails with a ValueError when passed yaml
Yaml is not valid json
"""
with self.assertRaises(ValueError):
actual = cfn_flip.to_yaml(self.input_yaml)
def generate_cluster(self):
self.__validate_parameters()
self._setup_network(self.configuration['vpc'])
self._create_log_group()
self._add_cluster_outputs()
self._add_cluster_parameters()
self._add_mappings()
self._add_metadata()
self._add_cluster()
return to_yaml(json.dumps(self.template.to_dict(), cls=DecimalEncoder))
def write_template(**stack_args):
cfn_json_path = 'templates_generated/json/{}.json'.format(
stack_args['StackName'])
cfn_yaml_path = 'templates_generated/yml/{}.yml'.format(
stack_args['StackName'])
with open(cfn_json_path, 'wt') as f:
f.write(stack_args['TemplateBody'])
logger.info('wrote json template')
with open(cfn_yaml_path, 'wt') as f:
f.write(cfn_flip.to_yaml(stack_args['TemplateBody']))
logger.info('wrote yml template')
def from_json(json):
try:
yaml = cfn_flip.to_yaml(json)
except Exception as e:
return user_error(e.message)
return {
"headers": {
"Content-Type": CONTENT_TYPE_YAML,
},
"body": yaml,
}
def write_tfstate_template(dest: Path) -> None:
"""Write TfState blueprint as a YAML CFN template.
Args:
dest: File to be written to.
"""
LOGGER.debug('writing TfState as a YAML template to "%s"', dest)
dest.write_text(
to_yaml(
TfState(
"test",
CfnginContext(environment={"namespace": "test"})).to_json()))
def write_tfstate_template(dest):
# type: (Path) -> None
"""Write TfState blueprint as a YAML CFN template.
Args:
dest (Path): File to be written to.
"""
LOGGER.debug('writing TfState as a YAML template to "%s"', dest)
# TODO remove use of six.u when dripping python 2 support
dest.write_text(six.u(to_yaml(TfState('test',
CFNginContext({'namespace': 'test'}),
None).to_json())))
def test_to_yaml_with_json(self):
"""
Test that to_yaml performs correctly
"""
actual = cfn_flip.to_yaml(self.input_json)
# The result should not parse as json
with self.assertRaises(ValueError):
json.loads(actual)
parsed_actual = custom_yaml.load(actual)
self.assertDictEqual(parsed_actual, self.parsed_yaml)
def test_to_yaml_with_json(input_json, parsed_yaml):
"""
Test that to_yaml performs correctly
"""
actual = cfn_flip.to_yaml(input_json)
# The result should not parse as json
with pytest.raises(ValueError):
load_json(actual)
parsed_actual = load_yaml(actual)
assert parsed_actual == parsed_yaml
def test_to_yaml_with_json(self):
"""
Test that to_yaml performs correctly
"""
actual = cfn_flip.to_yaml(self.input_json)
# The result should not parse as json
with self.assertRaises(ValueError):
json.loads(actual)
parsed_actual = yaml.load(actual)
self.assertDictEqual(parsed_actual, self.parsed_yaml)
def test_flip_to_yaml_with_multi_level_getatt():
"""
Test that we correctly convert multi-level Fn::GetAtt
from JSON to YAML format
"""
data = """
{
"Fn::GetAtt": ["First", "Second", "Third"]
}
"""
expected = "!GetAtt 'First.Second.Third'\n"
assert cfn_flip.to_yaml(data) == expected
def test_flip_to_yaml_with_newlines():
"""
Test that strings containing newlines are quoted
"""
source = r'["a", "b\n", "c\r\n", "d\r"]'
expected = "".join([
'- a\n',
'- "b\\n"\n',
'- "c\\r\\n"\n',
'- "d\\r"\n',
])
assert cfn_flip.to_yaml(source) == expected
def test_flip_to_yaml_with_multi_level_getatt(self):
"""
Test that we correctly convert multi-level Fn::GetAtt
from JSON to YAML format
"""
data = """
{
"Fn::GetAtt": ["First", "Second", "Third"]
}
"""
expected = "!GetAtt 'First.Second.Third'\n"
self.assertEqual(cfn_flip.to_yaml(data), expected)
def generate_cluster(self):
self.__validate_parameters()
self._setup_network(
self.configuration['vpc']['cidr'],
self.configuration['vpc']['subnets'],
self.configuration['vpc']['nat-gateway']
['elastic-ip-allocation-id'],
)
self._create_log_group()
self._add_cluster_outputs()
self._add_cluster_parameters()
self._add_mappings()
self._add_metadata()
self._add_cluster()
return to_yaml(json.dumps(self.template.to_dict(), cls=DecimalEncoder))
def test_flip_to_yaml_with_longhand_functions(input_json, parsed_json):
"""
When converting to yaml, sometimes we'll want to keep the long form
"""
actual1 = cfn_flip.flip(input_json, long_form=True)
actual2 = cfn_flip.to_yaml(input_json, long_form=True)
# No custom loader as there should be no custom tags
parsed_actual1 = yaml.load(actual1)
parsed_actual2 = yaml.load(actual2)
# We use the parsed JSON as it contains long form function calls
assert parsed_actual1 == parsed_json
assert parsed_actual2 == parsed_json
def test_to_yaml_with_long_json(input_long_json):
"""
Test that to_yaml performs correctly
"""
actual = cfn_flip.to_yaml(input_long_json)
# The result should not parse as json
with pytest.raises(ValueError):
load_json(actual)
parsed_actual = load_yaml(actual)
assert parsed_actual['TooShort'] == "foo\nbar\nbaz\nquuux"
assert 'WideText: >-' in actual
assert 'TooShort: "foo' in actual
def test_unconverted_types():
"""
When converting to yaml, we need to make sure all short-form types are tagged
"""
fns = {
"Fn::GetAtt": "!GetAtt",
"Fn::Sub": "!Sub",
"Ref": "!Ref",
"Condition": "!Condition",
}
for fn, tag in fns.items():
value = dump_json({fn: "something"})
expected = "{} 'something'\n".format(tag)
assert cfn_flip.to_yaml(value) == expected
def to_yaml(self, clean_up=False, long_form=False):
return cfn_flip.to_yaml(self.to_json(), clean_up=clean_up,
long_form=long_form)
def to_yaml(self):
return cfn_flip.to_yaml(self.to_json())
def to_yaml(self, long_form=False):
return cfn_flip.to_yaml(self.to_json(), long_form)