def _validate_delivery_snapshot_properties(self, properties): if not self.config_schema: self.config_schema = AWSServiceSpec( path="data/config/2014-11-12/service-2.json") # Verify that the deliveryFrequency is set to an acceptable value: if (properties.get("deliveryFrequency", None) not in self.config_schema.shapes["MaximumExecutionFrequency"] ["enum"]): raise InvalidDeliveryFrequency( properties.get("deliveryFrequency", None), self.config_schema.shapes["MaximumExecutionFrequency"]["enum"], )
def _validate_resource_types(self, resource_list): if not self.config_schema: self.config_schema = AWSServiceSpec( path="data/config/2014-11-12/service-2.json") # Verify that each entry exists in the supported list: bad_list = [] for resource in resource_list: # For PY2: r_str = str(resource) if r_str not in self.config_schema.shapes["ResourceType"]["enum"]: bad_list.append(r_str) if bad_list: raise InvalidResourceTypeException( bad_list, self.config_schema.shapes["ResourceType"]["enum"])
class ElasticMapReduceResponse(BaseResponse): # EMR end points are inconsistent in the placement of region name # in the URL, so parsing it out needs to be handled differently region_regex = [ re.compile(r'elasticmapreduce\.(.+?)\.amazonaws\.com'), re.compile(r'(.+?)\.elasticmapreduce\.amazonaws\.com') ] aws_service_spec = AWSServiceSpec('data/emr/2009-03-31/service-2.json') def get_region_from_url(self, request, full_url): parsed = urlparse(full_url) for regex in self.region_regex: match = regex.search(parsed.netloc) if match: return match.group(1) return self.default_region @property def backend(self): return emr_backends[self.region] @generate_boto3_response('AddInstanceGroups') def add_instance_groups(self): jobflow_id = self._get_param('JobFlowId') instance_groups = self._get_list_prefix('InstanceGroups.member') for item in instance_groups: item['instance_count'] = int(item['instance_count']) instance_groups = self.backend.add_instance_groups( jobflow_id, instance_groups) template = self.response_template(ADD_INSTANCE_GROUPS_TEMPLATE) return template.render(instance_groups=instance_groups) @generate_boto3_response('AddJobFlowSteps') def add_job_flow_steps(self): job_flow_id = self._get_param('JobFlowId') steps = self.backend.add_job_flow_steps( job_flow_id, steps_from_query_string(self._get_list_prefix('Steps.member'))) template = self.response_template(ADD_JOB_FLOW_STEPS_TEMPLATE) return template.render(steps=steps) @generate_boto3_response('AddTags') def add_tags(self): cluster_id = self._get_param('ResourceId') tags = tags_from_query_string(self.querystring) self.backend.add_tags(cluster_id, tags) template = self.response_template(ADD_TAGS_TEMPLATE) return template.render() def cancel_steps(self): raise NotImplementedError def create_security_configuration(self): raise NotImplementedError def delete_security_configuration(self): raise NotImplementedError @generate_boto3_response('DescribeCluster') def describe_cluster(self): cluster_id = self._get_param('ClusterId') cluster = self.backend.get_cluster(cluster_id) template = self.response_template(DESCRIBE_CLUSTER_TEMPLATE) return template.render(cluster=cluster) @generate_boto3_response('DescribeJobFlows') def describe_job_flows(self): created_after = self._get_param('CreatedAfter') created_before = self._get_param('CreatedBefore') job_flow_ids = self._get_multi_param("JobFlowIds.member") job_flow_states = self._get_multi_param('JobFlowStates.member') clusters = self.backend.describe_job_flows(job_flow_ids, job_flow_states, created_after, created_before) template = self.response_template(DESCRIBE_JOB_FLOWS_TEMPLATE) return template.render(clusters=clusters) def describe_security_configuration(self): raise NotImplementedError @generate_boto3_response('DescribeStep') def describe_step(self): cluster_id = self._get_param('ClusterId') step_id = self._get_param('StepId') step = self.backend.describe_step(cluster_id, step_id) template = self.response_template(DESCRIBE_STEP_TEMPLATE) return template.render(step=step) @generate_boto3_response('ListBootstrapActions') def list_bootstrap_actions(self): cluster_id = self._get_param('ClusterId') marker = self._get_param('Marker') bootstrap_actions, marker = self.backend.list_bootstrap_actions( cluster_id, marker) template = self.response_template(LIST_BOOTSTRAP_ACTIONS_TEMPLATE) return template.render(bootstrap_actions=bootstrap_actions, marker=marker) @generate_boto3_response('ListClusters') def list_clusters(self): cluster_states = self._get_multi_param('ClusterStates.member') created_after = self._get_param('CreatedAfter') created_before = self._get_param('CreatedBefore') marker = self._get_param('Marker') clusters, marker = self.backend.list_clusters(cluster_states, created_after, created_before, marker) template = self.response_template(LIST_CLUSTERS_TEMPLATE) return template.render(clusters=clusters, marker=marker) @generate_boto3_response('ListInstanceGroups') def list_instance_groups(self): cluster_id = self._get_param('ClusterId') marker = self._get_param('Marker') instance_groups, marker = self.backend.list_instance_groups( cluster_id, marker=marker) template = self.response_template(LIST_INSTANCE_GROUPS_TEMPLATE) return template.render(instance_groups=instance_groups, marker=marker) def list_instances(self): raise NotImplementedError @generate_boto3_response('ListSteps') def list_steps(self): cluster_id = self._get_param('ClusterId') marker = self._get_param('Marker') step_ids = self._get_multi_param('StepIds.member') step_states = self._get_multi_param('StepStates.member') steps, marker = self.backend.list_steps(cluster_id, marker=marker, step_ids=step_ids, step_states=step_states) template = self.response_template(LIST_STEPS_TEMPLATE) return template.render(steps=steps, marker=marker) @generate_boto3_response('ModifyInstanceGroups') def modify_instance_groups(self): instance_groups = self._get_list_prefix('InstanceGroups.member') for item in instance_groups: item['instance_count'] = int(item['instance_count']) instance_groups = self.backend.modify_instance_groups(instance_groups) template = self.response_template(MODIFY_INSTANCE_GROUPS_TEMPLATE) return template.render(instance_groups=instance_groups) @generate_boto3_response('RemoveTags') def remove_tags(self): cluster_id = self._get_param('ResourceId') tag_keys = self._get_multi_param('TagKeys.member') self.backend.remove_tags(cluster_id, tag_keys) template = self.response_template(REMOVE_TAGS_TEMPLATE) return template.render() @generate_boto3_response('RunJobFlow') def run_job_flow(self): instance_attrs = dict( master_instance_type=self._get_param( 'Instances.MasterInstanceType'), slave_instance_type=self._get_param('Instances.SlaveInstanceType'), instance_count=self._get_int_param('Instances.InstanceCount', 1), ec2_key_name=self._get_param('Instances.Ec2KeyName'), ec2_subnet_id=self._get_param('Instances.Ec2SubnetId'), hadoop_version=self._get_param('Instances.HadoopVersion'), availability_zone=self._get_param( 'Instances.Placement.AvailabilityZone', self.backend.region_name + 'a'), keep_job_flow_alive_when_no_steps=self._get_bool_param( 'Instances.KeepJobFlowAliveWhenNoSteps', False), termination_protected=self._get_bool_param( 'Instances.TerminationProtected', False), emr_managed_master_security_group=self._get_param( 'Instances.EmrManagedMasterSecurityGroup'), emr_managed_slave_security_group=self._get_param( 'Instances.EmrManagedSlaveSecurityGroup'), service_access_security_group=self._get_param( 'Instances.ServiceAccessSecurityGroup'), additional_master_security_groups=self._get_multi_param( 'Instances.AdditionalMasterSecurityGroups.member.'), additional_slave_security_groups=self._get_multi_param( 'Instances.AdditionalSlaveSecurityGroups.member.')) kwargs = dict( name=self._get_param('Name'), log_uri=self._get_param('LogUri'), job_flow_role=self._get_param('JobFlowRole'), service_role=self._get_param('ServiceRole'), steps=steps_from_query_string( self._get_list_prefix('Steps.member')), visible_to_all_users=self._get_bool_param('VisibleToAllUsers', False), instance_attrs=instance_attrs, ) bootstrap_actions = self._get_list_prefix('BootstrapActions.member') if bootstrap_actions: for ba in bootstrap_actions: args = [] idx = 1 keyfmt = 'script_bootstrap_action._args.member.{0}' key = keyfmt.format(idx) while key in ba: args.append(ba.pop(key)) idx += 1 key = keyfmt.format(idx) ba['args'] = args ba['script_path'] = ba.pop('script_bootstrap_action._path') kwargs['bootstrap_actions'] = bootstrap_actions configurations = self._get_list_prefix('Configurations.member') if configurations: for idx, config in enumerate(configurations, 1): for key in list(config.keys()): if key.startswith('properties.'): config.pop(key) config['properties'] = {} map_items = self._get_map_prefix( 'Configurations.member.{0}.Properties.entry'.format(idx)) config['properties'] = map_items kwargs['configurations'] = configurations release_label = self._get_param('ReleaseLabel') ami_version = self._get_param('AmiVersion') if release_label: kwargs['release_label'] = release_label if ami_version: message = ( 'Only one AMI version and release label may be specified. ' 'Provided AMI: {0}, release label: {1}.').format( ami_version, release_label) raise EmrError(error_type="ValidationException", message=message, template='error_json') else: if ami_version: kwargs['requested_ami_version'] = ami_version kwargs['running_ami_version'] = ami_version else: kwargs['running_ami_version'] = '1.0.0' cluster = self.backend.run_job_flow(**kwargs) applications = self._get_list_prefix('Applications.member') if applications: self.backend.add_applications(cluster.id, applications) else: self.backend.add_applications(cluster.id, [{ 'Name': 'Hadoop', 'Version': '0.18' }]) instance_groups = self._get_list_prefix( 'Instances.InstanceGroups.member') if instance_groups: for ig in instance_groups: ig['instance_count'] = int(ig['instance_count']) self.backend.add_instance_groups(cluster.id, instance_groups) tags = self._get_list_prefix('Tags.member') if tags: self.backend.add_tags(cluster.id, dict((d['key'], d['value']) for d in tags)) template = self.response_template(RUN_JOB_FLOW_TEMPLATE) return template.render(cluster=cluster) @generate_boto3_response('SetTerminationProtection') def set_termination_protection(self): termination_protection = self._get_param('TerminationProtected') job_ids = self._get_multi_param('JobFlowIds.member') self.backend.set_termination_protection(job_ids, termination_protection) template = self.response_template(SET_TERMINATION_PROTECTION_TEMPLATE) return template.render() @generate_boto3_response('SetVisibleToAllUsers') def set_visible_to_all_users(self): visible_to_all_users = self._get_param('VisibleToAllUsers') job_ids = self._get_multi_param('JobFlowIds.member') self.backend.set_visible_to_all_users(job_ids, visible_to_all_users) template = self.response_template(SET_VISIBLE_TO_ALL_USERS_TEMPLATE) return template.render() @generate_boto3_response('TerminateJobFlows') def terminate_job_flows(self): job_ids = self._get_multi_param('JobFlowIds.member.') self.backend.terminate_job_flows(job_ids) template = self.response_template(TERMINATE_JOB_FLOWS_TEMPLATE) return template.render()
class ElasticMapReduceResponse(BaseResponse): # EMR end points are inconsistent in the placement of region name # in the URL, so parsing it out needs to be handled differently region_regex = [ re.compile(r"elasticmapreduce\.(.+?)\.amazonaws\.com"), re.compile(r"(.+?)\.elasticmapreduce\.amazonaws\.com"), ] aws_service_spec = AWSServiceSpec("data/emr/2009-03-31/service-2.json") def get_region_from_url(self, request, full_url): parsed = urlparse(full_url) for regex in self.region_regex: match = regex.search(parsed.netloc) if match: return match.group(1) return self.default_region @property def backend(self): return emr_backends[self.region] @generate_boto3_response("AddInstanceGroups") def add_instance_groups(self): jobflow_id = self._get_param("JobFlowId") instance_groups = self._get_list_prefix("InstanceGroups.member") for item in instance_groups: item["instance_count"] = int(item["instance_count"]) instance_groups = self.backend.add_instance_groups( jobflow_id, instance_groups) template = self.response_template(ADD_INSTANCE_GROUPS_TEMPLATE) return template.render(instance_groups=instance_groups) @generate_boto3_response("AddJobFlowSteps") def add_job_flow_steps(self): job_flow_id = self._get_param("JobFlowId") steps = self.backend.add_job_flow_steps( job_flow_id, steps_from_query_string(self._get_list_prefix("Steps.member"))) template = self.response_template(ADD_JOB_FLOW_STEPS_TEMPLATE) return template.render(steps=steps) @generate_boto3_response("AddTags") def add_tags(self): cluster_id = self._get_param("ResourceId") tags = tags_from_query_string(self.querystring) self.backend.add_tags(cluster_id, tags) template = self.response_template(ADD_TAGS_TEMPLATE) return template.render() def cancel_steps(self): raise NotImplementedError def create_security_configuration(self): raise NotImplementedError def delete_security_configuration(self): raise NotImplementedError @generate_boto3_response("DescribeCluster") def describe_cluster(self): cluster_id = self._get_param("ClusterId") cluster = self.backend.get_cluster(cluster_id) template = self.response_template(DESCRIBE_CLUSTER_TEMPLATE) return template.render(cluster=cluster) @generate_boto3_response("DescribeJobFlows") def describe_job_flows(self): created_after = self._get_param("CreatedAfter") created_before = self._get_param("CreatedBefore") job_flow_ids = self._get_multi_param("JobFlowIds.member") job_flow_states = self._get_multi_param("JobFlowStates.member") clusters = self.backend.describe_job_flows(job_flow_ids, job_flow_states, created_after, created_before) template = self.response_template(DESCRIBE_JOB_FLOWS_TEMPLATE) return template.render(clusters=clusters) def describe_security_configuration(self): raise NotImplementedError @generate_boto3_response("DescribeStep") def describe_step(self): cluster_id = self._get_param("ClusterId") step_id = self._get_param("StepId") step = self.backend.describe_step(cluster_id, step_id) template = self.response_template(DESCRIBE_STEP_TEMPLATE) return template.render(step=step) @generate_boto3_response("ListBootstrapActions") def list_bootstrap_actions(self): cluster_id = self._get_param("ClusterId") marker = self._get_param("Marker") bootstrap_actions, marker = self.backend.list_bootstrap_actions( cluster_id, marker) template = self.response_template(LIST_BOOTSTRAP_ACTIONS_TEMPLATE) return template.render(bootstrap_actions=bootstrap_actions, marker=marker) @generate_boto3_response("ListClusters") def list_clusters(self): cluster_states = self._get_multi_param("ClusterStates.member") created_after = self._get_param("CreatedAfter") created_before = self._get_param("CreatedBefore") marker = self._get_param("Marker") clusters, marker = self.backend.list_clusters(cluster_states, created_after, created_before, marker) template = self.response_template(LIST_CLUSTERS_TEMPLATE) return template.render(clusters=clusters, marker=marker) @generate_boto3_response("ListInstanceGroups") def list_instance_groups(self): cluster_id = self._get_param("ClusterId") marker = self._get_param("Marker") instance_groups, marker = self.backend.list_instance_groups( cluster_id, marker=marker) template = self.response_template(LIST_INSTANCE_GROUPS_TEMPLATE) return template.render(instance_groups=instance_groups, marker=marker) def list_instances(self): raise NotImplementedError @generate_boto3_response("ListSteps") def list_steps(self): cluster_id = self._get_param("ClusterId") marker = self._get_param("Marker") step_ids = self._get_multi_param("StepIds.member") step_states = self._get_multi_param("StepStates.member") steps, marker = self.backend.list_steps(cluster_id, marker=marker, step_ids=step_ids, step_states=step_states) template = self.response_template(LIST_STEPS_TEMPLATE) return template.render(steps=steps, marker=marker) @generate_boto3_response("ModifyInstanceGroups") def modify_instance_groups(self): instance_groups = self._get_list_prefix("InstanceGroups.member") for item in instance_groups: item["instance_count"] = int(item["instance_count"]) instance_groups = self.backend.modify_instance_groups(instance_groups) template = self.response_template(MODIFY_INSTANCE_GROUPS_TEMPLATE) return template.render(instance_groups=instance_groups) @generate_boto3_response("RemoveTags") def remove_tags(self): cluster_id = self._get_param("ResourceId") tag_keys = self._get_multi_param("TagKeys.member") self.backend.remove_tags(cluster_id, tag_keys) template = self.response_template(REMOVE_TAGS_TEMPLATE) return template.render() @generate_boto3_response("RunJobFlow") def run_job_flow(self): instance_attrs = dict( main_instance_type=self._get_param("Instances.MainInstanceType"), subordinate_instance_type=self._get_param( "Instances.SubordinateInstanceType"), instance_count=self._get_int_param("Instances.InstanceCount", 1), ec2_key_name=self._get_param("Instances.Ec2KeyName"), ec2_subnet_id=self._get_param("Instances.Ec2SubnetId"), hadoop_version=self._get_param("Instances.HadoopVersion"), availability_zone=self._get_param( "Instances.Placement.AvailabilityZone", self.backend.region_name + "a"), keep_job_flow_alive_when_no_steps=self._get_bool_param( "Instances.KeepJobFlowAliveWhenNoSteps", False), termination_protected=self._get_bool_param( "Instances.TerminationProtected", False), emr_managed_main_security_group=self._get_param( "Instances.EmrManagedMainSecurityGroup"), emr_managed_subordinate_security_group=self._get_param( "Instances.EmrManagedSubordinateSecurityGroup"), service_access_security_group=self._get_param( "Instances.ServiceAccessSecurityGroup"), additional_main_security_groups=self._get_multi_param( "Instances.AdditionalMainSecurityGroups.member."), additional_subordinate_security_groups=self._get_multi_param( "Instances.AdditionalSubordinateSecurityGroups.member."), ) kwargs = dict( name=self._get_param("Name"), log_uri=self._get_param("LogUri"), job_flow_role=self._get_param("JobFlowRole"), service_role=self._get_param("ServiceRole"), steps=steps_from_query_string( self._get_list_prefix("Steps.member")), visible_to_all_users=self._get_bool_param("VisibleToAllUsers", False), instance_attrs=instance_attrs, ) bootstrap_actions = self._get_list_prefix("BootstrapActions.member") if bootstrap_actions: for ba in bootstrap_actions: args = [] idx = 1 keyfmt = "script_bootstrap_action._args.member.{0}" key = keyfmt.format(idx) while key in ba: args.append(ba.pop(key)) idx += 1 key = keyfmt.format(idx) ba["args"] = args ba["script_path"] = ba.pop("script_bootstrap_action._path") kwargs["bootstrap_actions"] = bootstrap_actions configurations = self._get_list_prefix("Configurations.member") if configurations: for idx, config in enumerate(configurations, 1): for key in list(config.keys()): if key.startswith("properties."): config.pop(key) config["properties"] = {} map_items = self._get_map_prefix( "Configurations.member.{0}.Properties.entry".format(idx)) config["properties"] = map_items kwargs["configurations"] = configurations release_label = self._get_param("ReleaseLabel") ami_version = self._get_param("AmiVersion") if release_label: kwargs["release_label"] = release_label if ami_version: message = ( "Only one AMI version and release label may be specified. " "Provided AMI: {0}, release label: {1}.").format( ami_version, release_label) raise EmrError( error_type="ValidationException", message=message, template="error_json", ) else: if ami_version: kwargs["requested_ami_version"] = ami_version kwargs["running_ami_version"] = ami_version else: kwargs["running_ami_version"] = "1.0.0" custom_ami_id = self._get_param("CustomAmiId") if custom_ami_id: kwargs["custom_ami_id"] = custom_ami_id if release_label and release_label < "emr-5.7.0": message = "Custom AMI is not allowed" raise EmrError( error_type="ValidationException", message=message, template="error_json", ) elif ami_version: message = "Custom AMI is not supported in this version of EMR" raise EmrError( error_type="ValidationException", message=message, template="error_json", ) cluster = self.backend.run_job_flow(**kwargs) applications = self._get_list_prefix("Applications.member") if applications: self.backend.add_applications(cluster.id, applications) else: self.backend.add_applications(cluster.id, [{ "Name": "Hadoop", "Version": "0.18" }]) instance_groups = self._get_list_prefix( "Instances.InstanceGroups.member") if instance_groups: for ig in instance_groups: ig["instance_count"] = int(ig["instance_count"]) self.backend.add_instance_groups(cluster.id, instance_groups) tags = self._get_list_prefix("Tags.member") if tags: self.backend.add_tags(cluster.id, dict((d["key"], d["value"]) for d in tags)) template = self.response_template(RUN_JOB_FLOW_TEMPLATE) return template.render(cluster=cluster) @generate_boto3_response("SetTerminationProtection") def set_termination_protection(self): termination_protection = self._get_param("TerminationProtected") job_ids = self._get_multi_param("JobFlowIds.member") self.backend.set_termination_protection(job_ids, termination_protection) template = self.response_template(SET_TERMINATION_PROTECTION_TEMPLATE) return template.render() @generate_boto3_response("SetVisibleToAllUsers") def set_visible_to_all_users(self): visible_to_all_users = self._get_param("VisibleToAllUsers") job_ids = self._get_multi_param("JobFlowIds.member") self.backend.set_visible_to_all_users(job_ids, visible_to_all_users) template = self.response_template(SET_VISIBLE_TO_ALL_USERS_TEMPLATE) return template.render() @generate_boto3_response("TerminateJobFlows") def terminate_job_flows(self): job_ids = self._get_multi_param("JobFlowIds.member.") self.backend.terminate_job_flows(job_ids) template = self.response_template(TERMINATE_JOB_FLOWS_TEMPLATE) return template.render()
def test_flatten_json_request_body(): spec = AWSServiceSpec("data/emr/2009-03-31/service-2.json").input_spec("RunJobFlow") body = { "Name": "cluster", "Instances": { "Ec2KeyName": "ec2key", "InstanceGroups": [ {"InstanceRole": "MASTER", "InstanceType": "m1.small"}, {"InstanceRole": "CORE", "InstanceType": "m1.medium"}, ], "Placement": {"AvailabilityZone": "us-east-1"}, }, "Steps": [ { "HadoopJarStep": { "Properties": [ {"Key": "k1", "Value": "v1"}, {"Key": "k2", "Value": "v2"}, ], "Args": ["arg1", "arg2"], } } ], "Configurations": [ { "Classification": "class", "Properties": {"propkey1": "propkey1", "propkey2": "propkey2"}, }, {"Classification": "anotherclass", "Properties": {"propkey3": "propkey3"}}, ], } flat = flatten_json_request_body("", body, spec) flat["Name"].should.equal(body["Name"]) flat["Instances.Ec2KeyName"].should.equal(body["Instances"]["Ec2KeyName"]) for idx in range(2): flat[ "Instances.InstanceGroups.member." + str(idx + 1) + ".InstanceRole" ].should.equal(body["Instances"]["InstanceGroups"][idx]["InstanceRole"]) flat[ "Instances.InstanceGroups.member." + str(idx + 1) + ".InstanceType" ].should.equal(body["Instances"]["InstanceGroups"][idx]["InstanceType"]) flat["Instances.Placement.AvailabilityZone"].should.equal( body["Instances"]["Placement"]["AvailabilityZone"] ) for idx in range(1): prefix = "Steps.member." + str(idx + 1) + ".HadoopJarStep" step = body["Steps"][idx]["HadoopJarStep"] i = 0 while prefix + ".Properties.member." + str(i + 1) + ".Key" in flat: flat[prefix + ".Properties.member." + str(i + 1) + ".Key"].should.equal( step["Properties"][i]["Key"] ) flat[prefix + ".Properties.member." + str(i + 1) + ".Value"].should.equal( step["Properties"][i]["Value"] ) i += 1 i = 0 while prefix + ".Args.member." + str(i + 1) in flat: flat[prefix + ".Args.member." + str(i + 1)].should.equal(step["Args"][i]) i += 1 for idx in range(2): flat["Configurations.member." + str(idx + 1) + ".Classification"].should.equal( body["Configurations"][idx]["Classification"] ) props = {} i = 1 keyfmt = "Configurations.member.{0}.Properties.entry.{1}" key = keyfmt.format(idx + 1, i) while key + ".key" in flat: props[flat[key + ".key"]] = flat[key + ".value"] i += 1 key = keyfmt.format(idx + 1, i) props.should.equal(body["Configurations"][idx]["Properties"])
def test_flatten_json_request_body(): spec = AWSServiceSpec('data/emr/2009-03-31/service-2.json').input_spec( 'RunJobFlow') body = { 'Name': 'cluster', 'Instances': { 'Ec2KeyName': 'ec2key', 'InstanceGroups': [ { 'InstanceRole': 'MASTER', 'InstanceType': 'm1.small' }, { 'InstanceRole': 'CORE', 'InstanceType': 'm1.medium' }, ], 'Placement': { 'AvailabilityZone': 'us-east-1' }, }, 'Steps': [ { 'HadoopJarStep': { 'Properties': [{ 'Key': 'k1', 'Value': 'v1' }, { 'Key': 'k2', 'Value': 'v2' }], 'Args': ['arg1', 'arg2'] } }, ], 'Configurations': [ { 'Classification': 'class', 'Properties': { 'propkey1': 'propkey1', 'propkey2': 'propkey2' } }, { 'Classification': 'anotherclass', 'Properties': { 'propkey3': 'propkey3' } }, ] } flat = flatten_json_request_body('', body, spec) flat['Name'].should.equal(body['Name']) flat['Instances.Ec2KeyName'].should.equal(body['Instances']['Ec2KeyName']) for idx in range(2): flat['Instances.InstanceGroups.member.' + str(idx + 1) + '.InstanceRole'].should.equal( body['Instances']['InstanceGroups'][idx]['InstanceRole']) flat['Instances.InstanceGroups.member.' + str(idx + 1) + '.InstanceType'].should.equal( body['Instances']['InstanceGroups'][idx]['InstanceType']) flat['Instances.Placement.AvailabilityZone'].should.equal( body['Instances']['Placement']['AvailabilityZone']) for idx in range(1): prefix = 'Steps.member.' + str(idx + 1) + '.HadoopJarStep' step = body['Steps'][idx]['HadoopJarStep'] i = 0 while prefix + '.Properties.member.' + str(i + 1) + '.Key' in flat: flat[prefix + '.Properties.member.' + str(i + 1) + '.Key'].should.equal(step['Properties'][i]['Key']) flat[prefix + '.Properties.member.' + str(i + 1) + '.Value'].should.equal(step['Properties'][i]['Value']) i += 1 i = 0 while prefix + '.Args.member.' + str(i + 1) in flat: flat[prefix + '.Args.member.' + str(i + 1)].should.equal( step['Args'][i]) i += 1 for idx in range(2): flat['Configurations.member.' + str(idx + 1) + '.Classification'].should.equal( body['Configurations'][idx]['Classification']) props = {} i = 1 keyfmt = 'Configurations.member.{0}.Properties.entry.{1}' key = keyfmt.format(idx + 1, i) while key + '.key' in flat: props[flat[key + '.key']] = flat[key + '.value'] i += 1 key = keyfmt.format(idx + 1, i) props.should.equal(body['Configurations'][idx]['Properties'])