def test_json_errors_parsing(): # The outputs/ directory has sample output responses # For each file in outputs/ there's a corresponding file # in expected/ that has the expected parsed response. base_dir = os.path.join(os.path.dirname(__file__), 'json') json_responses_dir = os.path.join(base_dir, 'errors') expected_parsed_dir = os.path.join(base_dir, 'expected') session = botocore.session.get_session() for json_response_file in os.listdir(json_responses_dir): # Files look like: 'datapipeline-create-pipeline.json' service_name, operation_name = os.path.splitext( json_response_file)[0].split('-', 1) expected_parsed_response = os.path.join(expected_parsed_dir, json_response_file) raw_response_file = os.path.join(json_responses_dir, json_response_file) with open(expected_parsed_response) as f: expected = json.load(f) service_model = session.get_service_model(service_name) operation_names = service_model.operation_names operation_model = None for op_name in operation_names: if xform_name(op_name) == operation_name.replace('-', '_'): operation_model = service_model.operation_model(op_name) with open(raw_response_file, 'rb') as f: raw_response_body = f.read() yield _test_parsed_response, raw_response_file, \ raw_response_body, operation_model, expected
def test_valid(self): session = botocore.session.get_session() for impl in common.SimpleDescribe.__subclasses__(): if issubclass(impl, self.ignore): continue if getattr(impl, 'describe_action', None) is None: continue # API use must be versioned assert len(impl.api_version or '') == 10, '{} is not versioned'.format(impl) # Not allowed to set Describe.describe_filters AND implement Describe.get_describe_filters is_df_set = bool(impl.describe_filters is not None) is_gdf_set = 'get_describe_filters' in impl.__dict__ assert (is_gdf_set and not is_df_set) or not is_gdf_set service = session.get_service_model(impl.service_name, impl.api_version) methods = {xform_name(s): s for s in service.operation_names} operation = service.operation_model(methods[impl.describe_action]) if impl.describe_envelope in ['@', '[@]']: continue if impl.describe_envelope.startswith('['): continue if '.' not in impl.describe_envelope and ':' not in impl.describe_envelope: assert impl.describe_envelope in operation.output_shape.members
def _all_inputs(): session = botocore.session.get_session() for service_name in session.get_available_services(): service_model = session.get_service_model(service_name) for operation_name in service_model.operation_names: operation_model = service_model.operation_model(operation_name) input_shape = operation_model.input_shape if input_shape is not None and input_shape.members: yield input_shape, service_name, operation_name
def _pagination_configs(): session = botocore.session.get_session() loader = session.get_component('data_loader') services = loader.list_available_services('paginators-1') for service_name in services: service_model = session.get_service_model(service_name) page_config = loader.load_service_model(service_name, 'paginators-1', service_model.api_version) for op_name, single_config in page_config['pagination'].items(): yield (op_name, single_config, service_model)
def test_can_generate_all_inputs(): session = botocore.session.get_session() generator = ArgumentGenerator() for service_name in session.get_available_services(): service_model = session.get_service_model(service_name) for operation_name in service_model.operation_names: operation_model = service_model.operation_model(operation_name) input_shape = operation_model.input_shape if input_shape is not None and input_shape.members: yield (_test_can_generate_skeleton, generator, input_shape, service_name, operation_name)
def _shared_example_configs(): session = botocore.session.Session() loader = session.get_component('data_loader') services = loader.list_available_services('examples-1') for service in services: service_model = session.get_service_model(service) example_config = loader.load_service_model(service, 'examples-1', service_model.api_version) examples = example_config.get("examples", {}) for operation, operation_examples in examples.items(): for example in operation_examples: yield operation, example, service_model
def test_lint_shared_example_configs(): session = botocore.session.Session() loader = session.get_component('data_loader') services = loader.list_available_services('examples-1') for service in services: service_model = session.get_service_model(service) example_config = loader.load_service_model( service, 'examples-1', service_model.api_version ) examples = example_config.get("examples", {}) for operation, operation_examples in examples.items(): for example in operation_examples: yield _lint_single_example, operation, example, service_model
def test_lint_pagination_configs(): session = botocore.session.get_session() loader = session.get_component('data_loader') services = loader.list_available_services('paginators-1') for service_name in services: service_model = session.get_service_model(service_name) page_config = loader.load_service_model(service_name, 'paginators-1', service_model.api_version) for op_name, single_config in page_config['pagination'].items(): yield ( _lint_single_paginator, op_name, single_config, service_model )
def _xml_test_cases(): for dp in ['responses', 'errors']: data_path = os.path.join(os.path.dirname(__file__), 'xml') data_path = os.path.join(data_path, dp) session = create_session() xml_files = glob.glob('%s/*.xml' % data_path) service_names = set() for fn in xml_files: service_names.add(os.path.split(fn)[1].split('-')[0]) for service_name in service_names: service_model = session.get_service_model(service_name) service_xml_files = glob.glob('%s/%s-*.xml' % (data_path, service_name)) for xmlfile in service_xml_files: expected = _get_expected_parsed_result(xmlfile) operation_model = _get_operation_model(service_model, xmlfile) raw_response_body = _get_raw_response_body(xmlfile) yield xmlfile, raw_response_body, operation_model, expected
def test_xml_parsing(): for dp in ['responses', 'errors']: data_path = os.path.join(os.path.dirname(__file__), 'xml') data_path = os.path.join(data_path, dp) session = create_session() xml_files = glob.glob('%s/*.xml' % data_path) service_names = set() for fn in xml_files: service_names.add(os.path.split(fn)[1].split('-')[0]) for service_name in service_names: service_model = session.get_service_model(service_name) service_xml_files = glob.glob('%s/%s-*.xml' % (data_path, service_name)) for xmlfile in service_xml_files: expected = _get_expected_parsed_result(xmlfile) operation_model = _get_operation_model(service_model, xmlfile) raw_response_body = _get_raw_response_body(xmlfile) yield _test_parsed_response, xmlfile, raw_response_body, \ operation_model, expected
def _uhg_test_json_parsing(): input_path = os.path.join(os.path.dirname(__file__), 'json') input_path = os.path.join(input_path, 'inputs') output_path = os.path.join(os.path.dirname(__file__), 'json') output_path = os.path.join(output_path, 'outputs') session = botocore.session.get_session() jsonfiles = glob.glob('%s/*.json' % input_path) service_names = set() for fn in jsonfiles: service_names.add(os.path.split(fn)[1].split('-')[0]) for service_name in service_names: service_model = session.get_service_model(service_name) service_json_files = glob.glob('%s/%s-*.json' % (input_path, service_name)) for jsonfile in service_json_files: expected = _get_expected_parsed_result(jsonfile) operation_model = _get_operation_model(service_model, jsonfile) with open(jsonfile, 'rb') as f: raw_response_body = f.read() yield _test_parsed_response, jsonfile, \ raw_response_body, operation_model, expected
def docs_for(service_name): """ Generate reference documentation (low and high level) for a service by name. This generates docs for the latest available version. :type service_name: string :param service_name: The service short-name, like 'ec2' :rtype: string """ session = botocore.session.get_session() service_model = session.get_service_model(service_name) print('Processing {0}-{1}'.format(service_name, service_model.api_version)) official_name = service_model.metadata.get( 'serviceAbbreviation', service_model.metadata.get('serviceFullName')) docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name)) docs += '.. contents:: Table of Contents\n\n' docs += document_client(service_name, official_name, service_model) filename = (os.path.dirname(__file__) + '/data/resources/' '{0}-{1}.resources.json').format(service_name, service_model.api_version) if os.path.exists(filename): data = json.load(open(filename)) model = ResourceModel(service_name, data['service'], data['resources']) docs += document_resource(service_name, official_name, model, service_model) for name, model in sorted(data['resources'].items(), key=lambda i:i[0]): resource_model = ResourceModel(name, model, data['resources']) docs += document_resource(service_name, official_name, resource_model, service_model) return docs
def docs_for(service_name, session=None, resource_filename=None): """ Generate reference documentation (low and high level) for a service by name. This generates docs for the latest available version. :type service_name: string :param service_name: The service short-name, like 'ec2' :type session: botocore.session.Session :param session: Existing pre-setup session or ``None``. :type resource_filename: string :param resource_filename: Full path to the service's resource JSON description file. If unset, then this is loaded from the default location. :rtype: string """ if session is None: session = botocore.session.get_session() service_model = session.get_service_model(service_name) print('Processing {0}-{1}'.format(service_name, service_model.api_version)) # The following creates an official name of 'Amazon Simple Storage # Service (S3)' our of 'Amazon Simple Storage Service' and 'Amazon S3'. # It tries to be smart, so for `Amazon DynamoDB' and 'DynamoDB' we would # get an official name of 'Amazon DynamoDB'. official_name = service_model.metadata.get('serviceFullName') short_name = service_model.metadata.get('serviceAbbreviation', '') if short_name.startswith('Amazon'): short_name = short_name[7:] if short_name.startswith('AWS'): short_name = short_name[4:] if short_name and short_name.lower() not in official_name.lower(): official_name += ' ({0})'.format(short_name) docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name)) docs += '.. contents:: Table of Contents\n :depth: 2\n\n' # TODO: Get this information from the model somehow in the future. # For now creating and introspecing a client is a quick and # dirty way to access waiters/paginators. client = session.create_client(service_name, aws_access_key_id='dummy', aws_secret_access_key='dummy', region_name='us-east-1') docs += document_client(service_name, official_name, service_model, client) docs += document_client_waiter(session, official_name, service_name, service_model, client) if resource_filename is None: resource_filename = (os.path.dirname(__file__) + '/data/resources/' '{0}-{1}.resources.json').format( service_name, service_model.api_version) # We can't use a set here because dicts aren't hashable! models = {} if os.path.exists(resource_filename): data = json.load(open(resource_filename)) model = ResourceModel(service_name, data['service'], data['resources']) for collection_model in model.collections: collection_model.parent_name = model.name models[collection_model.name] = { 'type': 'collection', 'model': collection_model } docs += document_resource(service_name, official_name, model, service_model, session) # First, collect all the models... for name, model in sorted(data['resources'].items(), key=lambda i: i[0]): resource_model = ResourceModel(name, model, data['resources']) shape = None if resource_model.shape: shape = service_model.shape_for(resource_model.shape) resource_model.load_rename_map(shape) if name not in models: models[name] = {'type': 'resource', 'model': resource_model} for collection_model in resource_model.collections: collection_model.parent_name = xform_name(resource_model.name) cname = collection_model.name + 'CollectionManager' if cname not in models: models[cname] = { 'type': 'collection', 'model': collection_model } # Then render them out in alphabetical order. for name, item in sorted(models.items()): model = item['model'] if item['type'] == 'resource': docs += document_resource(service_name, official_name, model, service_model, session) elif item['type'] == 'collection': docs += document_collection(service_name, official_name, model, model.resource.model, service_model) return docs
def docs_for(service_name): """ Generate reference documentation (low and high level) for a service by name. This generates docs for the latest available version. :type service_name: string :param service_name: The service short-name, like 'ec2' :rtype: string """ session = botocore.session.get_session() service_model = session.get_service_model(service_name) print('Processing {0}-{1}'.format(service_name, service_model.api_version)) # The following creates an official name of 'Amazon Simple Storage # Service (S3)' our of 'Amazon Simple Storage Service' and 'Amazon S3'. # It tries to be smart, so for `Amazon DynamoDB' and 'DynamoDB' we would # get an official name of 'Amazon DynamoDB'. official_name = service_model.metadata.get('serviceFullName') short_name = service_model.metadata.get('serviceAbbreviation', '') if short_name.startswith('Amazon'): short_name = short_name[7:] if short_name.startswith('AWS'): short_name = short_name[4:] if short_name and short_name.lower() not in official_name.lower(): official_name += ' ({0})'.format(short_name) docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name)) docs += '.. contents:: Table of Contents\n :depth: 2\n\n' docs += document_client(service_name, official_name, service_model) docs += document_client_waiter(session, official_name, service_name, service_model) filename = (os.path.dirname(__file__) + '/data/resources/' '{0}-{1}.resources.json').format(service_name, service_model.api_version) # We can't use a set here because dicts aren't hashable! models = {} if os.path.exists(filename): data = json.load(open(filename)) model = ResourceModel(service_name, data['service'], data['resources']) for collection_model in model.collections: collection_model.parent_name = model.name models[collection_model.name] = { 'type': 'collection', 'model': collection_model } docs += document_resource(service_name, official_name, model, service_model, session) # First, collect all the models... for name, model in sorted(data['resources'].items(), key=lambda i: i[0]): resource_model = ResourceModel(name, model, data['resources']) shape = None if resource_model.shape: shape = service_model.shape_for(resource_model.shape) resource_model.load_rename_map(shape) if name not in models: models[name] = {'type': 'resource', 'model': resource_model} for collection_model in resource_model.collections: collection_model.parent_name = xform_name(resource_model.name) cname = collection_model.name + 'CollectionManager' if cname not in models: models[cname] = { 'type': 'collection', 'model': collection_model } # Then render them out in alphabetical order. for name, item in sorted(models.items()): model = item['model'] if item['type'] == 'resource': docs += document_resource(service_name, official_name, model, service_model, session) elif item['type'] == 'collection': docs += document_collection(service_name, official_name, model, model.resource.model, service_model) return docs
def docs_for(service_name, session=None, resource_filename=None): """ Generate reference documentation (low and high level) for a service by name. This generates docs for the latest available version. :type service_name: string :param service_name: The service short-name, like 'ec2' :type session: botocore.session.Session :param session: Existing pre-setup session or ``None``. :type resource_filename: string :param resource_filename: Full path to the service's resource JSON description file. If unset, then this is loaded from the default location. :rtype: string """ if session is None: session = botocore.session.get_session() service_model = session.get_service_model(service_name) print('Processing {0}-{1}'.format(service_name, service_model.api_version)) # The following creates an official name of 'Amazon Simple Storage # Service (S3)' our of 'Amazon Simple Storage Service' and 'Amazon S3'. # It tries to be smart, so for `Amazon DynamoDB' and 'DynamoDB' we would # get an official name of 'Amazon DynamoDB'. official_name = service_model.metadata.get('serviceFullName') short_name = service_model.metadata.get('serviceAbbreviation', '') if short_name.startswith('Amazon'): short_name = short_name[7:] if short_name.startswith('AWS'): short_name = short_name[4:] if short_name and short_name.lower() not in official_name.lower(): official_name += ' ({0})'.format(short_name) docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name)) docs += '.. contents:: Table of Contents\n :depth: 2\n\n' # TODO: Get this information from the model somehow in the future. # For now creating and introspecing a client is a quick and # dirty way to access waiters/paginators. client = session.create_client(service_name, aws_access_key_id='dummy', aws_secret_access_key='dummy', region_name='us-east-1') docs += document_client(service_name, official_name, service_model, client) docs += document_client_waiter(session, official_name, service_name, service_model, client) if resource_filename is None: resource_filename = (os.path.dirname(__file__) + '/data/resources/' '{0}-{1}.resources.json').format( service_name, service_model.api_version) # We can't use a set here because dicts aren't hashable! models = {} if os.path.exists(resource_filename): data = json.load(open(resource_filename)) model = ResourceModel(service_name, data['service'], data['resources']) for collection_model in model.collections: collection_model.parent_name = model.name models[collection_model.name] = { 'type': 'collection', 'model': collection_model } docs += document_resource(service_name, official_name, model, service_model, session) # First, collect all the models... for name, model in sorted(data['resources'].items(), key=lambda i:i[0]): resource_model = ResourceModel(name, model, data['resources']) shape = None if resource_model.shape: shape = service_model.shape_for(resource_model.shape) resource_model.load_rename_map(shape) if name not in models: models[name] = {'type': 'resource', 'model': resource_model} for collection_model in resource_model.collections: collection_model.parent_name = xform_name(resource_model.name) cname = collection_model.name + 'CollectionManager' if cname not in models: models[cname] = {'type': 'collection', 'model': collection_model} # Then render them out in alphabetical order. for name, item in sorted(models.items()): model = item['model'] if item['type'] == 'resource': docs += document_resource(service_name, official_name, model, service_model, session) elif item['type'] == 'collection': docs += document_collection( service_name, official_name, model, model.resource.model, service_model) return docs
def docs_for(service_name): """ Generate reference documentation (low and high level) for a service by name. This generates docs for the latest available version. :type service_name: string :param service_name: The service short-name, like 'ec2' :rtype: string """ session = botocore.session.get_session() service_model = session.get_service_model(service_name) print('Processing {0}-{1}'.format(service_name, service_model.api_version)) # The following creates an official name of 'Amazon Simple Storage # Service (S3)' our of 'Amazon Simple Storage Service' and 'Amazon S3'. # It tries to be smart, so for `Amazon DynamoDB' and 'DynamoDB' we would # get an official name of 'Amazon DynamoDB'. official_name = service_model.metadata.get('serviceFullName') short_name = service_model.metadata.get('serviceAbbreviation', '') if short_name.startswith('Amazon'): short_name = short_name[7:] if short_name.startswith('AWS'): short_name = short_name[4:] if short_name and short_name.lower() not in official_name.lower(): official_name += ' ({0})'.format(short_name) docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name)) docs += '.. contents:: Table of Contents\n\n' docs += document_client(service_name, official_name, service_model) filename = (os.path.dirname(__file__) + '/data/resources/' '{0}-{1}.resources.json').format(service_name, service_model.api_version) # We can't use a set here because dicts aren't hashable! models = {} if os.path.exists(filename): data = json.load(open(filename)) model = ResourceModel(service_name, data['service'], data['resources']) for collection_model in model.collections: collection_model.parent_name = model.name models[collection_model.name] = { 'type': 'collection', 'model': collection_model } docs += document_resource(service_name, official_name, model, service_model) # First, collect all the models... for name, model in sorted(data['resources'].items(), key=lambda i:i[0]): resource_model = ResourceModel(name, model, data['resources']) if name not in models: models[name] = {'type': 'resource', 'model': resource_model} for collection_model in resource_model.collections: collection_model.parent_name = xform_name(resource_model.name) cname = collection_model.name + 'CollectionManager' if cname not in models: models[cname] = {'type': 'collection', 'model': collection_model} # Then render them out in alphabetical order. for name, item in sorted(models.items()): model = item['model'] if item['type'] == 'resource': docs += document_resource(service_name, official_name, model, service_model) elif item['type'] == 'collection': docs += document_collection( service_name, official_name, model, model.resource.model, service_model) return docs