def test_getatt_from_yaml(): """ Test that we correctly convert the short form of GetAtt into the correct JSON format from YAML """ source = """ - !GetAtt foo.bar - Fn::GetAtt: [foo, bar] """ expected = [ { "Fn::GetAtt": ["foo", "bar"] }, { "Fn::GetAtt": ["foo", "bar"] }, ] # No clean actual = cfn_flip.to_json(source, clean_up=False) assert load_json(actual) == expected # With clean actual = cfn_flip.to_json(source, clean_up=True) assert load_json(actual) == expected
def test_flip_to_yaml(input_json, input_yaml, parsed_yaml): """ Test that flip performs correctly transforming from json to yaml """ actual = cfn_flip.flip(input_json) assert actual == input_yaml + "\n" # The result should not parse as json with pytest.raises(ValueError): load_json(actual) parsed_actual = load_yaml(actual) assert parsed_actual == parsed_yaml
def test_to_yaml_with_json(input_json, parsed_yaml): """ Test that to_yaml performs correctly """ actual = cfn_flip.to_yaml(input_json) # The result should not parse as json with pytest.raises(ValueError): load_json(actual) parsed_actual = load_yaml(actual) assert parsed_actual == parsed_yaml
def test_flip_to_clean_yaml(input_json, clean_yaml, parsed_clean_yaml): """ Test that flip performs correctly transforming from json to yaml and the `clean_up` flag is active """ actual = cfn_flip.flip(input_json, clean_up=True) assert actual == clean_yaml + "\n" # The result should not parse as json with pytest.raises(ValueError): load_json(actual) parsed_actual = load_yaml(actual) assert parsed_actual == parsed_clean_yaml
def test_dump_json(): """ JSON dumping just needs to know about datetimes, provide a nice indent, and preserve order """ import sys source = ODict(( ("z", datetime.time(3, 45)), ("m", datetime.date(2012, 5, 2)), ("a", datetime.datetime(2012, 5, 2, 3, 45)), )) actual = dump_json(source) assert load_json(actual) == { "z": "03:45:00", "m": "2012-05-02", "a": "2012-05-02T03:45:00", } if sys.version_info < (3, 6): fail_message = r"\(1\+1j\) is not JSON serializable" elif sys.version_info < (3, 7): fail_message = "Object of type 'complex' is not JSON serializable" else: fail_message = "Object of type complex is not JSON serializable" with pytest.raises(TypeError, match=fail_message): dump_json({ "c": 1 + 1j, })
def test_to_yaml_with_long_json(input_long_json): """ Test that to_yaml performs correctly """ actual = cfn_flip.to_yaml(input_long_json) # The result should not parse as json with pytest.raises(ValueError): load_json(actual) parsed_actual = load_yaml(actual) assert parsed_actual['TooShort'] == "foo\nbar\nbaz\nquuux" assert 'WideText: >-' in actual assert 'TooShort: "foo' in actual
def test_to_json_with_yaml(input_yaml, parsed_json): """ Test that to_json performs correctly """ actual = cfn_flip.to_json(input_yaml) assert load_json(actual) == parsed_json
def test_flip_to_multibyte_yaml(multibyte_yaml, parsed_multibyte_json): """ Test that load multibyte file performs correctly """ actual = cfn_flip.to_json(multibyte_yaml) assert load_json(actual) == parsed_multibyte_json
def test_flip_to_json_with_condition(): """ Test that the Condition key is correctly converted """ source = """ MyAndCondition: !And - !Equals ["sg-mysggroup", !Ref "ASecurityGroup"] - !Condition SomeOtherCondition """ expected = { "MyAndCondition": { "Fn::And": [{ "Fn::Equals": ["sg-mysggroup", { "Ref": "ASecurityGroup" }] }, { "Condition": "SomeOtherCondition" }] } } actual = cfn_flip.to_json(source, clean_up=True) assert load_json(actual) == expected
def test_flip_to_json(input_yaml, input_json, parsed_json): """ Test that flip performs correctly transforming from yaml to json """ actual = cfn_flip.flip(input_yaml) assert load_json(actual) == parsed_json
def test_no_flip_with_json(input_json, parsed_json): """ We should be able to submit JSON and get JSON back """ actual = cfn_flip.flip(input_json, no_flip=True) assert load_json(actual) == parsed_json
def test_flip_with_json_output(input_yaml, parsed_json): """ We should be able to specify that the output is JSON """ actual = cfn_flip.flip(input_yaml, out_format="json") assert load_json(actual) == parsed_json
def test_no_flip_with_explicit_json(input_json, parsed_json): """ We should be able to submit JSON and get JSON back and specify the output format explicity """ actual = cfn_flip.flip(input_json, out_format="json", no_flip=True) assert load_json(actual) == parsed_json
def test_flip_to_clean_json(input_yaml, clean_json, parsed_clean_json): """ Test that flip performs correctly transforming from yaml to json and the `clean_up` flag is active """ actual = cfn_flip.flip(input_yaml, clean_up=True) assert load_json(actual) == parsed_clean_json
def test_to_json_with_json(input_json, parsed_json): """ Test that to_json still works when passed json (All json is valid yaml) """ actual = cfn_flip.to_json(input_json) assert load_json(actual) == parsed_json
def to_yaml(template, clean_up=False, long_form=False): """ Assume the input is JSON and convert to YAML """ data = load_json(template) if clean_up: data = clean(data) return dump_yaml(data, clean_up, long_form)
def load(template): """ Try to guess the input format """ try: data = load_json(template) return data, "json" except ValueError: data = load_yaml(template) return data, "yaml"
def test_flip_to_json_with_multi_level_getatt(): """ Test that we correctly convert multi-level Fn::GetAtt from YAML to JSON format """ data = "!GetAtt 'First.Second.Third'\n" expected = {"Fn::GetAtt": ["First", "Second.Third"]} actual = cfn_flip.to_json(data, clean_up=True) assert load_json(actual) == expected
def yaml_to_json(p_input_file): json_out = ROOT_DIR+'/../../../tacocat_cf.json' with open( p_input_file,'r') as yaml_in: yaml_data = load_yaml(yaml_in) yaml_data = clean(yaml_data) with open(json_out,'w') as out_file: json_data = dump_json(yaml_data) json_data = load_json(json_data) json.dump(json_data ,out_file, sort_keys=True, indent=4, separators=(',', ': ')) print('*--*--*--*--*--*--*--*--*--*--*--*--**--*--*--*--*--*--*--*--*--*--*--*') print('The new JSON file has been created in the root folder of the project with the name tacocat_cf.json') print('*--*--*--*--*--*--*--*--*--*--*--*--**--*--*--*--*--*--*--*--*--*--*--*')
def flip(template, in_format=None, out_format=None, clean_up=False, no_flip=False, long_form=False): """ Figure out the input format and convert the data to the opposing output format """ # Do we need to figure out the input format? if not in_format: # Load the template as JSON? if (out_format == "json" and no_flip) or (out_format == "yaml" and not no_flip): in_format = "json" elif (out_format == "yaml" and no_flip) or (out_format == "json" and not no_flip): in_format = "yaml" # Load the data if in_format == "json": data = load_json(template) elif in_format == "yaml": data = load_yaml(template) else: data, in_format = load(template) # Clean up? if clean_up: data = clean(data) # Figure out the output format if not out_format: if (in_format == "json" and no_flip) or (in_format == "yaml" and not no_flip): out_format = "json" else: out_format = "yaml" # Finished! if out_format == "json": if sys.version[0] == "3": return dump_json(data) else: return dump_json(data).encode('utf-8') return dump_yaml(data, clean_up, long_form)
def test_flip_to_json_with_datetimes(): """ Test that the json encoder correctly handles dates and datetimes """ tricky_data = """ a date: 2017-03-02 a datetime: 2017-03-02 19:52:00 """ actual = cfn_flip.to_json(tricky_data) parsed_actual = load_json(actual) assert parsed_actual == { "a date": "2017-03-02", "a datetime": "2017-03-02T19:52:00", }
def convert_yaml_to_json(yaml_file): converted_file = ROOT_DIR + '/../../../tacocat_cf.json' with open(yaml_file, 'r') as file_in: yaml_data = load_yaml(file_in) yaml_data = clean(yaml_data) with open(converted_file, 'w') as file_out: json_data = dump_json(yaml_data) json_data = load_json(json_data) json.dump(json_data, file_out, sort_keys=True, indent=4, separators=(',', ': ')) print( '#####--Converted successfully, please check the root folder with the name tacocat_cf.json--#####' )
def test_load_json(): """ Should map to an ordered dict """ source = """ { "z": "first", "m": "middle", "a": "last" } """ actual = load_json(source) assert type(actual) == ODict assert list(actual.keys()) == ["z", "m", "a"] assert actual["z"] == "first" assert actual["m"] == "middle" assert actual["a"] == "last"
def flip(template, out_format=None, clean_up=False, no_flip=False, long_form=False): """ Figure out the input format and convert the data to the opposing output format """ data = None in_format = None if no_flip: in_format = out_format elif out_format == "json": in_format = "yaml" elif out_format == "yaml": in_format = "json" if in_format == "json": data = load_json(template) elif in_format == "yaml": data = load_yaml(template) else: try: data, in_format = load(template) except Exception: raise Exception("Could not determine the input format") if no_flip: out_format = in_format elif in_format == "json": out_format = "yaml" else: out_format = "json" if clean_up: data = clean(data) if out_format == "json": return dump_json(data) return dump_yaml(data, clean_up, long_form)
def test_dump_json(): """ JSON dumping just needs to know about datetimes, provide a nice indent, and preserve order """ source = ODict(( ("z", datetime.time(3, 45)), ("m", datetime.date(2012, 5, 2)), ("a", datetime.datetime(2012, 5, 2, 3, 45)), )) actual = dump_json(source) assert load_json(actual) == { "z": "03:45:00", "m": "2012-05-02", "a": "2012-05-02T03:45:00", } with pytest.raises(TypeError, message="complex is not JSON serializable"): dump_json({ "c": 1 + 1j, })
def run(self): stack = self.ensure_stack_is_in_complete_status() status = stack.get("StackStatus") with self.spoke_regional_client("cloudformation") as cloudformation: if status == "ROLLBACK_COMPLETE": if self.should_delete_rollback_complete_stacks: cloudformation.ensure_deleted( StackName=self.stack_name_to_use) else: raise Exception( f"Stack: {self.stack_name_to_use} is in ROLLBACK_COMPLETE and need remediation" ) task_output = dict( **self.params_for_results_display(), account_parameters=tasks.unwrap(self.account_parameters), launch_parameters=tasks.unwrap(self.launch_parameters), manifest_parameters=tasks.unwrap(self.manifest_parameters), ) all_params = self.get_parameter_values() template_to_provision_source = self.input().get("template").open( "r").read() try: template_to_provision = cfn_tools.load_yaml( template_to_provision_source) except Exception: try: template_to_provision = cfn_tools.load_json( template_to_provision_source) except Exception: raise Exception("Could not parse new template as YAML or JSON") params_to_use = dict() for param_name, p in template_to_provision.get("Parameters", {}).items(): if all_params.get(param_name, p.get("DefaultValue")) is not None: params_to_use[param_name] = all_params.get( param_name, p.get("DefaultValue")) existing_stack_params_dict = dict() existing_template = "" if status in [ "CREATE_COMPLETE", "UPDATE_ROLLBACK_COMPLETE", "UPDATE_COMPLETE", "IMPORT_COMPLETE", "IMPORT_ROLLBACK_COMPLETE", ]: with self.spoke_regional_client( "cloudformation") as cloudformation: existing_stack_params_dict = {} summary_response = cloudformation.get_template_summary( StackName=self.stack_name_to_use, ) for parameter in summary_response.get("Parameters"): existing_stack_params_dict[parameter.get( "ParameterKey")] = parameter.get("DefaultValue") for stack_param in stack.get("Parameters", []): existing_stack_params_dict[stack_param.get( "ParameterKey")] = stack_param.get("ParameterValue") template_body = cloudformation.get_template( StackName=self.stack_name_to_use, TemplateStage="Original").get("TemplateBody") try: existing_template = cfn_tools.load_yaml(template_body) except Exception: try: existing_template = cfn_tools.load_json(template_body) except Exception: raise Exception( "Could not parse existing template as YAML or JSON" ) template_to_use = cfn_tools.dump_yaml(template_to_provision) if status == "UPDATE_ROLLBACK_COMPLETE": need_to_provision = True else: if existing_stack_params_dict == params_to_use: self.info(f"params unchanged") if template_to_use == cfn_tools.dump_yaml(existing_template): self.info(f"template the same") need_to_provision = False else: self.info(f"template changed") need_to_provision = True else: self.info(f"params changed") need_to_provision = True if need_to_provision: provisioning_parameters = [] for p in params_to_use.keys(): provisioning_parameters.append({ "ParameterKey": p, "ParameterValue": params_to_use.get(p) }) with self.spoke_regional_client( "cloudformation") as cloudformation: a = dict( StackName=self.stack_name_to_use, TemplateBody=template_to_use, ShouldUseChangeSets=False, Capabilities=self.capabilities, Parameters=provisioning_parameters, ShouldDeleteRollbackComplete=self. should_delete_rollback_complete_stacks, ) if self.use_service_role: a["RoleARN"] = config.get_puppet_stack_role_arn( self.account_id) cloudformation.create_or_update(**a) task_output["provisioned"] = need_to_provision self.info(f"self.execution is {self.execution}") if self.execution == constants.EXECUTION_MODE_HUB: self.info( f"Running in execution mode: {self.execution}, checking for SSM outputs" ) if len(self.ssm_param_outputs) > 0: with self.spoke_regional_client( "cloudformation") as spoke_cloudformation: stack_details = aws.get_stack_output_for( spoke_cloudformation, self.stack_name_to_use, ) for ssm_param_output in self.ssm_param_outputs: self.info( f"writing SSM Param: {ssm_param_output.get('stack_output')}" ) with self.hub_client("ssm") as ssm: found_match = False # TODO push into another task for output in stack_details.get("Outputs", []): if output.get("OutputKey") == ssm_param_output.get( "stack_output"): ssm_parameter_name = ssm_param_output.get( "param_name") ssm_parameter_name = ssm_parameter_name.replace( "${AWS::Region}", self.region) ssm_parameter_name = ssm_parameter_name.replace( "${AWS::AccountId}", self.account_id) found_match = True ssm.put_parameter_and_wait( Name=ssm_parameter_name, Value=output.get("OutputValue"), Type=ssm_param_output.get( "param_type", "String"), Overwrite=True, ) if not found_match: raise Exception( f"[{self.uid}] Could not find match for {ssm_param_output.get('stack_output')}" ) self.write_output(task_output) else: self.write_output(task_output)
def parsed_multibyte_json(): return load_json(multibyte_json())
def parsed_clean_json(): return load_json(clean_json())
def parsed_json(): return load_json(input_json())
def run(self): status = self.get_current_status() if status == "-": self.write_result( "-", self.version_id, effect=constants.CHANGE, current_status="-", active="N/A", notes="Stack would be created", ) elif status == "ROLLBACK_COMPLETE": if self.should_delete_rollback_complete_stacks: self.write_result( "-", self.version_id, effect=constants.CHANGE, current_status="-", active="N/A", notes="Stack would be replaced", ) else: self.write_result( "-", "-", effect=constants.NO_CHANGE, current_status="-", active="N/A", notes="Stack needs remediation - it's in ROLLBACK_COMPLETE", ) else: task_output = dict( **self.params_for_results_display(), account_parameters=tasks.unwrap(self.account_parameters), launch_parameters=tasks.unwrap(self.launch_parameters), manifest_parameters=tasks.unwrap(self.manifest_parameters), ) all_params = self.get_parameter_values() template_to_provision_source = self.input().get("template").open( "r").read() try: template_to_provision = cfn_tools.load_yaml( template_to_provision_source) except Exception: try: template_to_provision = cfn_tools.load_json( template_to_provision_source) except Exception: raise Exception( "Could not parse new template as YAML or JSON") params_to_use = dict() for param_name, p in template_to_provision.get("Parameters", {}).items(): if all_params.get(param_name, p.get("DefaultValue")) is not None: params_to_use[param_name] = all_params.get( param_name, p.get("DefaultValue")) existing_stack_params_dict = dict() existing_template = "" if status in [ "CREATE_COMPLETE", "UPDATE_ROLLBACK_COMPLETE", "UPDATE_COMPLETE", "IMPORT_COMPLETE", "IMPORT_ROLLBACK_COMPLETE", ]: with self.spoke_regional_client( "cloudformation") as cloudformation: existing_stack_params_dict = {} stack = cloudformation.describe_stacks( StackName=self.stack_name).get("Stacks")[0] summary_response = cloudformation.get_template_summary( StackName=self.stack_name, ) for parameter in summary_response.get("Parameters"): existing_stack_params_dict[parameter.get( "ParameterKey")] = parameter.get("DefaultValue") for stack_param in stack.get("Parameters", []): existing_stack_params_dict[stack_param.get( "ParameterKey")] = stack_param.get( "ParameterValue") template_body = cloudformation.get_template( StackName=self.stack_name, TemplateStage="Original").get("TemplateBody") try: existing_template = cfn_tools.load_yaml(template_body) except Exception: try: existing_template = cfn_tools.load_json( template_body) except Exception: raise Exception( "Could not parse existing template as YAML or JSON" ) template_to_use = cfn_tools.dump_yaml(template_to_provision) if status == "UPDATE_ROLLBACK_COMPLETE": self.write_result( "?", self.version_id, effect=constants.CHANGE, current_status=status, active="N/A", notes="Stack would be updated", ) else: if existing_stack_params_dict == params_to_use: self.info(f"params unchanged") if template_to_use == cfn_tools.dump_yaml( existing_template): self.info(f"template the same") self.write_result( "?", self.version_id, effect=constants.NO_CHANGE, current_status=status, active="N/A", notes="No change", ) else: self.info(f"template changed") self.write_result( "?", self.version_id, effect=constants.CHANGE, current_status=status, active="N/A", notes="Template has changed", ) else: self.info(f"params changed") self.write_result( "?", self.version_id, effect=constants.CHANGE, current_status=status, active="N/A", notes="Parameters have changed", )