def test_json_errors_parsing():
    # The outputs/ directory has sample output responses
    # For each file in outputs/ there's a corresponding file
    # in expected/ that has the expected parsed response.
    base_dir = os.path.join(os.path.dirname(__file__), 'json')
    json_responses_dir = os.path.join(base_dir, 'errors')
    expected_parsed_dir = os.path.join(base_dir, 'expected')
    session = botocore.session.get_session()
    for json_response_file in os.listdir(json_responses_dir):
        # Files look like: 'datapipeline-create-pipeline.json'
        service_name, operation_name = os.path.splitext(
            json_response_file)[0].split('-', 1)
        expected_parsed_response = os.path.join(expected_parsed_dir,
                                                json_response_file)
        raw_response_file = os.path.join(json_responses_dir,
                                         json_response_file)
        with open(expected_parsed_response) as f:
            expected = json.load(f)
        service_model = session.get_service_model(service_name)
        operation_names = service_model.operation_names
        operation_model = None
        for op_name in operation_names:
            if xform_name(op_name) == operation_name.replace('-', '_'):
                operation_model = service_model.operation_model(op_name)
        with open(raw_response_file, 'rb') as f:
            raw_response_body = f.read()
        yield _test_parsed_response, raw_response_file, \
            raw_response_body, operation_model, expected
Esempio n. 2
0
    def test_valid(self):
        session = botocore.session.get_session()

        for impl in common.SimpleDescribe.__subclasses__():
            if issubclass(impl, self.ignore):
                continue

            if getattr(impl, 'describe_action', None) is None:
                continue

            # API use must be versioned
            assert len(impl.api_version or '') == 10, '{} is not versioned'.format(impl)

            # Not allowed to set Describe.describe_filters AND implement Describe.get_describe_filters
            is_df_set = bool(impl.describe_filters is not None)
            is_gdf_set = 'get_describe_filters' in impl.__dict__
            assert (is_gdf_set and not is_df_set) or not is_gdf_set

            service = session.get_service_model(impl.service_name, impl.api_version)
            methods = {xform_name(s): s for s in service.operation_names}
            operation = service.operation_model(methods[impl.describe_action])

            if impl.describe_envelope in ['@', '[@]']:
                continue

            if impl.describe_envelope.startswith('['):
                continue

            if '.' not in impl.describe_envelope and ':' not in impl.describe_envelope:
                assert impl.describe_envelope in operation.output_shape.members
def test_json_errors_parsing():
    # The outputs/ directory has sample output responses
    # For each file in outputs/ there's a corresponding file
    # in expected/ that has the expected parsed response.
    base_dir = os.path.join(os.path.dirname(__file__), 'json')
    json_responses_dir = os.path.join(base_dir, 'errors')
    expected_parsed_dir = os.path.join(base_dir, 'expected')
    session = botocore.session.get_session()
    for json_response_file in os.listdir(json_responses_dir):
        # Files look like: 'datapipeline-create-pipeline.json'
        service_name, operation_name = os.path.splitext(
            json_response_file)[0].split('-', 1)
        expected_parsed_response = os.path.join(expected_parsed_dir,
                                                json_response_file)
        raw_response_file = os.path.join(json_responses_dir,
                                         json_response_file)
        with open(expected_parsed_response) as f:
            expected = json.load(f)
        service_model = session.get_service_model(service_name)
        operation_names = service_model.operation_names
        operation_model = None
        for op_name in operation_names:
            if xform_name(op_name) == operation_name.replace('-', '_'):
                operation_model = service_model.operation_model(op_name)
        with open(raw_response_file, 'rb') as f:
            raw_response_body = f.read()
        yield _test_parsed_response, raw_response_file, \
            raw_response_body, operation_model, expected
Esempio n. 4
0
def _all_inputs():
    session = botocore.session.get_session()
    for service_name in session.get_available_services():
        service_model = session.get_service_model(service_name)
        for operation_name in service_model.operation_names:
            operation_model = service_model.operation_model(operation_name)
            input_shape = operation_model.input_shape
            if input_shape is not None and input_shape.members:
                yield input_shape, service_name, operation_name
Esempio n. 5
0
def _pagination_configs():
    session = botocore.session.get_session()
    loader = session.get_component('data_loader')
    services = loader.list_available_services('paginators-1')
    for service_name in services:
        service_model = session.get_service_model(service_name)
        page_config = loader.load_service_model(service_name, 'paginators-1',
                                                service_model.api_version)
        for op_name, single_config in page_config['pagination'].items():
            yield (op_name, single_config, service_model)
Esempio n. 6
0
def test_can_generate_all_inputs():
    session = botocore.session.get_session()
    generator = ArgumentGenerator()
    for service_name in session.get_available_services():
        service_model = session.get_service_model(service_name)
        for operation_name in service_model.operation_names:
            operation_model = service_model.operation_model(operation_name)
            input_shape = operation_model.input_shape
            if input_shape is not None and input_shape.members:
                yield (_test_can_generate_skeleton, generator,
                       input_shape, service_name, operation_name)
Esempio n. 7
0
def _shared_example_configs():
    session = botocore.session.Session()
    loader = session.get_component('data_loader')
    services = loader.list_available_services('examples-1')
    for service in services:
        service_model = session.get_service_model(service)
        example_config = loader.load_service_model(service, 'examples-1',
                                                   service_model.api_version)
        examples = example_config.get("examples", {})
        for operation, operation_examples in examples.items():
            for example in operation_examples:
                yield operation, example, service_model
def test_lint_shared_example_configs():
    session = botocore.session.Session()
    loader = session.get_component('data_loader')
    services = loader.list_available_services('examples-1')
    for service in services:
        service_model = session.get_service_model(service)
        example_config = loader.load_service_model(
            service, 'examples-1', service_model.api_version
        )
        examples = example_config.get("examples", {})
        for operation, operation_examples in examples.items():
            for example in operation_examples:
                yield _lint_single_example, operation, example, service_model
def test_lint_pagination_configs():
    session = botocore.session.get_session()
    loader = session.get_component('data_loader')
    services = loader.list_available_services('paginators-1')
    for service_name in services:
        service_model = session.get_service_model(service_name)
        page_config = loader.load_service_model(service_name,
                                                'paginators-1',
                                                service_model.api_version)
        for op_name, single_config in page_config['pagination'].items():
            yield (
                _lint_single_paginator,
                op_name,
                single_config,
                service_model
            )
Esempio n. 10
0
def _xml_test_cases():
    for dp in ['responses', 'errors']:
        data_path = os.path.join(os.path.dirname(__file__), 'xml')
        data_path = os.path.join(data_path, dp)
        session = create_session()
        xml_files = glob.glob('%s/*.xml' % data_path)
        service_names = set()
        for fn in xml_files:
            service_names.add(os.path.split(fn)[1].split('-')[0])
        for service_name in service_names:
            service_model = session.get_service_model(service_name)
            service_xml_files = glob.glob('%s/%s-*.xml' %
                                          (data_path, service_name))
            for xmlfile in service_xml_files:
                expected = _get_expected_parsed_result(xmlfile)
                operation_model = _get_operation_model(service_model, xmlfile)
                raw_response_body = _get_raw_response_body(xmlfile)
                yield xmlfile, raw_response_body, operation_model, expected
def test_xml_parsing():
    for dp in ['responses', 'errors']:
        data_path = os.path.join(os.path.dirname(__file__), 'xml')
        data_path = os.path.join(data_path, dp)
        session = create_session()
        xml_files = glob.glob('%s/*.xml' % data_path)
        service_names = set()
        for fn in xml_files:
            service_names.add(os.path.split(fn)[1].split('-')[0])
        for service_name in service_names:
            service_model = session.get_service_model(service_name)
            service_xml_files = glob.glob('%s/%s-*.xml' % (data_path,
                                                           service_name))
            for xmlfile in service_xml_files:
                expected = _get_expected_parsed_result(xmlfile)
                operation_model = _get_operation_model(service_model, xmlfile)
                raw_response_body = _get_raw_response_body(xmlfile)
                yield _test_parsed_response, xmlfile, raw_response_body, \
                    operation_model, expected
def _uhg_test_json_parsing():
    input_path = os.path.join(os.path.dirname(__file__), 'json')
    input_path = os.path.join(input_path, 'inputs')
    output_path = os.path.join(os.path.dirname(__file__), 'json')
    output_path = os.path.join(output_path, 'outputs')
    session = botocore.session.get_session()
    jsonfiles = glob.glob('%s/*.json' % input_path)
    service_names = set()
    for fn in jsonfiles:
        service_names.add(os.path.split(fn)[1].split('-')[0])
    for service_name in service_names:
        service_model = session.get_service_model(service_name)
        service_json_files = glob.glob('%s/%s-*.json' % (input_path,
                                                         service_name))
        for jsonfile in service_json_files:
            expected = _get_expected_parsed_result(jsonfile)
            operation_model = _get_operation_model(service_model, jsonfile)
            with open(jsonfile, 'rb') as f:
                raw_response_body = f.read()
            yield _test_parsed_response, jsonfile, \
                raw_response_body, operation_model, expected
def _uhg_test_json_parsing():
    input_path = os.path.join(os.path.dirname(__file__), 'json')
    input_path = os.path.join(input_path, 'inputs')
    output_path = os.path.join(os.path.dirname(__file__), 'json')
    output_path = os.path.join(output_path, 'outputs')
    session = botocore.session.get_session()
    jsonfiles = glob.glob('%s/*.json' % input_path)
    service_names = set()
    for fn in jsonfiles:
        service_names.add(os.path.split(fn)[1].split('-')[0])
    for service_name in service_names:
        service_model = session.get_service_model(service_name)
        service_json_files = glob.glob('%s/%s-*.json' %
                                       (input_path, service_name))
        for jsonfile in service_json_files:
            expected = _get_expected_parsed_result(jsonfile)
            operation_model = _get_operation_model(service_model, jsonfile)
            with open(jsonfile, 'rb') as f:
                raw_response_body = f.read()
            yield _test_parsed_response, jsonfile, \
                raw_response_body, operation_model, expected
Esempio n. 14
0
def docs_for(service_name):
    """
    Generate reference documentation (low and high level) for a service
    by name. This generates docs for the latest available version.

    :type service_name: string
    :param service_name: The service short-name, like 'ec2'
    :rtype: string
    """
    session = botocore.session.get_session()
    service_model = session.get_service_model(service_name)

    print('Processing {0}-{1}'.format(service_name, service_model.api_version))

    official_name = service_model.metadata.get(
        'serviceAbbreviation', service_model.metadata.get('serviceFullName'))

    docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name))

    docs += '.. contents:: Table of Contents\n\n'

    docs += document_client(service_name, official_name, service_model)

    filename = (os.path.dirname(__file__) + '/data/resources/'
                '{0}-{1}.resources.json').format(service_name,
                                                 service_model.api_version)
    if os.path.exists(filename):
        data = json.load(open(filename))
        model = ResourceModel(service_name, data['service'], data['resources'])

        docs += document_resource(service_name, official_name, model,
                                  service_model)

        for name, model in sorted(data['resources'].items(),
                                  key=lambda i:i[0]):
            resource_model = ResourceModel(name, model, data['resources'])
            docs += document_resource(service_name, official_name,
                                      resource_model, service_model)

    return docs
Esempio n. 15
0
def docs_for(service_name, session=None, resource_filename=None):
    """
    Generate reference documentation (low and high level) for a service
    by name. This generates docs for the latest available version.

    :type service_name: string
    :param service_name: The service short-name, like 'ec2'
    :type session: botocore.session.Session
    :param session: Existing pre-setup session or ``None``.
    :type resource_filename: string
    :param resource_filename: Full path to the service's resource JSON
                              description file. If unset, then this is
                              loaded from the default location.
    :rtype: string
    """
    if session is None:
        session = botocore.session.get_session()

    service_model = session.get_service_model(service_name)

    print('Processing {0}-{1}'.format(service_name, service_model.api_version))

    # The following creates an official name of 'Amazon Simple Storage
    # Service (S3)' our of 'Amazon Simple Storage Service' and 'Amazon S3'.
    # It tries to be smart, so for `Amazon DynamoDB' and 'DynamoDB' we would
    # get an official name of 'Amazon DynamoDB'.
    official_name = service_model.metadata.get('serviceFullName')
    short_name = service_model.metadata.get('serviceAbbreviation', '')
    if short_name.startswith('Amazon'):
        short_name = short_name[7:]
    if short_name.startswith('AWS'):
        short_name = short_name[4:]
    if short_name and short_name.lower() not in official_name.lower():
        official_name += ' ({0})'.format(short_name)

    docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name))

    docs += '.. contents:: Table of Contents\n   :depth: 2\n\n'

    # TODO: Get this information from the model somehow in the future.
    #       For now creating and introspecing a client is a quick and
    #       dirty way to access waiters/paginators.
    client = session.create_client(service_name,
                                   aws_access_key_id='dummy',
                                   aws_secret_access_key='dummy',
                                   region_name='us-east-1')

    docs += document_client(service_name, official_name, service_model, client)
    docs += document_client_waiter(session, official_name, service_name,
                                   service_model, client)

    if resource_filename is None:
        resource_filename = (os.path.dirname(__file__) + '/data/resources/'
                             '{0}-{1}.resources.json').format(
                                 service_name, service_model.api_version)
    # We can't use a set here because dicts aren't hashable!
    models = {}
    if os.path.exists(resource_filename):
        data = json.load(open(resource_filename))
        model = ResourceModel(service_name, data['service'], data['resources'])

        for collection_model in model.collections:
            collection_model.parent_name = model.name
            models[collection_model.name] = {
                'type': 'collection',
                'model': collection_model
            }

        docs += document_resource(service_name, official_name, model,
                                  service_model, session)

        # First, collect all the models...
        for name, model in sorted(data['resources'].items(),
                                  key=lambda i: i[0]):
            resource_model = ResourceModel(name, model, data['resources'])

            shape = None
            if resource_model.shape:
                shape = service_model.shape_for(resource_model.shape)
            resource_model.load_rename_map(shape)

            if name not in models:
                models[name] = {'type': 'resource', 'model': resource_model}

            for collection_model in resource_model.collections:
                collection_model.parent_name = xform_name(resource_model.name)

                cname = collection_model.name + 'CollectionManager'
                if cname not in models:
                    models[cname] = {
                        'type': 'collection',
                        'model': collection_model
                    }

        # Then render them out in alphabetical order.
        for name, item in sorted(models.items()):
            model = item['model']
            if item['type'] == 'resource':
                docs += document_resource(service_name, official_name, model,
                                          service_model, session)
            elif item['type'] == 'collection':
                docs += document_collection(service_name, official_name, model,
                                            model.resource.model,
                                            service_model)

    return docs
Esempio n. 16
0
def docs_for(service_name):
    """
    Generate reference documentation (low and high level) for a service
    by name. This generates docs for the latest available version.

    :type service_name: string
    :param service_name: The service short-name, like 'ec2'
    :rtype: string
    """
    session = botocore.session.get_session()
    service_model = session.get_service_model(service_name)

    print('Processing {0}-{1}'.format(service_name, service_model.api_version))

    # The following creates an official name of 'Amazon Simple Storage
    # Service (S3)' our of 'Amazon Simple Storage Service' and 'Amazon S3'.
    # It tries to be smart, so for `Amazon DynamoDB' and 'DynamoDB' we would
    # get an official name of 'Amazon DynamoDB'.
    official_name = service_model.metadata.get('serviceFullName')
    short_name = service_model.metadata.get('serviceAbbreviation', '')
    if short_name.startswith('Amazon'):
        short_name = short_name[7:]
    if short_name.startswith('AWS'):
        short_name = short_name[4:]
    if short_name and short_name.lower() not in official_name.lower():
        official_name += ' ({0})'.format(short_name)

    docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name))

    docs += '.. contents:: Table of Contents\n   :depth: 2\n\n'

    docs += document_client(service_name, official_name, service_model)
    docs += document_client_waiter(session, official_name, service_name,
                                   service_model)

    filename = (os.path.dirname(__file__) + '/data/resources/'
                '{0}-{1}.resources.json').format(service_name,
                                                 service_model.api_version)
    # We can't use a set here because dicts aren't hashable!
    models = {}
    if os.path.exists(filename):
        data = json.load(open(filename))
        model = ResourceModel(service_name, data['service'], data['resources'])

        for collection_model in model.collections:
            collection_model.parent_name = model.name
            models[collection_model.name] = {
                'type': 'collection',
                'model': collection_model
            }

        docs += document_resource(service_name, official_name, model,
                                  service_model, session)

        # First, collect all the models...
        for name, model in sorted(data['resources'].items(),
                                  key=lambda i: i[0]):
            resource_model = ResourceModel(name, model, data['resources'])

            shape = None
            if resource_model.shape:
                shape = service_model.shape_for(resource_model.shape)
            resource_model.load_rename_map(shape)

            if name not in models:
                models[name] = {'type': 'resource', 'model': resource_model}

            for collection_model in resource_model.collections:
                collection_model.parent_name = xform_name(resource_model.name)

                cname = collection_model.name + 'CollectionManager'
                if cname not in models:
                    models[cname] = {
                        'type': 'collection',
                        'model': collection_model
                    }

        # Then render them out in alphabetical order.
        for name, item in sorted(models.items()):
            model = item['model']
            if item['type'] == 'resource':
                docs += document_resource(service_name, official_name, model,
                                          service_model, session)
            elif item['type'] == 'collection':
                docs += document_collection(service_name, official_name, model,
                                            model.resource.model,
                                            service_model)

    return docs
Esempio n. 17
0
def docs_for(service_name, session=None, resource_filename=None):
    """
    Generate reference documentation (low and high level) for a service
    by name. This generates docs for the latest available version.

    :type service_name: string
    :param service_name: The service short-name, like 'ec2'
    :type session: botocore.session.Session
    :param session: Existing pre-setup session or ``None``.
    :type resource_filename: string
    :param resource_filename: Full path to the service's resource JSON
                              description file. If unset, then this is
                              loaded from the default location.
    :rtype: string
    """
    if session is None:
        session = botocore.session.get_session()

    service_model = session.get_service_model(service_name)

    print('Processing {0}-{1}'.format(service_name, service_model.api_version))

    # The following creates an official name of 'Amazon Simple Storage
    # Service (S3)' our of 'Amazon Simple Storage Service' and 'Amazon S3'.
    # It tries to be smart, so for `Amazon DynamoDB' and 'DynamoDB' we would
    # get an official name of 'Amazon DynamoDB'.
    official_name = service_model.metadata.get('serviceFullName')
    short_name = service_model.metadata.get('serviceAbbreviation', '')
    if short_name.startswith('Amazon'):
        short_name = short_name[7:]
    if short_name.startswith('AWS'):
        short_name = short_name[4:]
    if short_name and short_name.lower() not in official_name.lower():
        official_name += ' ({0})'.format(short_name)

    docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name))

    docs += '.. contents:: Table of Contents\n   :depth: 2\n\n'

    # TODO: Get this information from the model somehow in the future.
    #       For now creating and introspecing a client is a quick and
    #       dirty way to access waiters/paginators.
    client = session.create_client(service_name, aws_access_key_id='dummy',
                                   aws_secret_access_key='dummy',
                                   region_name='us-east-1')

    docs += document_client(service_name, official_name, service_model,
                            client)
    docs += document_client_waiter(session, official_name, service_name,
                                   service_model, client)

    if resource_filename is None:
        resource_filename = (os.path.dirname(__file__) + '/data/resources/'
                             '{0}-{1}.resources.json').format(
                                service_name, service_model.api_version)
    # We can't use a set here because dicts aren't hashable!
    models = {}
    if os.path.exists(resource_filename):
        data = json.load(open(resource_filename))
        model = ResourceModel(service_name, data['service'], data['resources'])

        for collection_model in model.collections:
            collection_model.parent_name = model.name
            models[collection_model.name] = {
                'type': 'collection',
                'model': collection_model
            }

        docs += document_resource(service_name, official_name, model,
                                  service_model, session)

        # First, collect all the models...
        for name, model in sorted(data['resources'].items(),
                                  key=lambda i:i[0]):
            resource_model = ResourceModel(name, model, data['resources'])

            shape = None
            if resource_model.shape:
                shape = service_model.shape_for(resource_model.shape)
            resource_model.load_rename_map(shape)

            if name not in models:
                models[name] = {'type': 'resource', 'model': resource_model}

            for collection_model in resource_model.collections:
                collection_model.parent_name = xform_name(resource_model.name)

                cname = collection_model.name + 'CollectionManager'
                if cname not in models:
                    models[cname] = {'type': 'collection',
                                     'model': collection_model}

        # Then render them out in alphabetical order.
        for name, item in sorted(models.items()):
            model = item['model']
            if item['type'] == 'resource':
                docs += document_resource(service_name, official_name,
                                          model, service_model, session)
            elif item['type'] == 'collection':
                docs += document_collection(
                    service_name, official_name, model,
                    model.resource.model, service_model)

    return docs
Esempio n. 18
0
def docs_for(service_name):
    """
    Generate reference documentation (low and high level) for a service
    by name. This generates docs for the latest available version.

    :type service_name: string
    :param service_name: The service short-name, like 'ec2'
    :rtype: string
    """
    session = botocore.session.get_session()
    service_model = session.get_service_model(service_name)

    print('Processing {0}-{1}'.format(service_name, service_model.api_version))

    # The following creates an official name of 'Amazon Simple Storage
    # Service (S3)' our of 'Amazon Simple Storage Service' and 'Amazon S3'.
    # It tries to be smart, so for `Amazon DynamoDB' and 'DynamoDB' we would
    # get an official name of 'Amazon DynamoDB'.
    official_name = service_model.metadata.get('serviceFullName')
    short_name = service_model.metadata.get('serviceAbbreviation', '')
    if short_name.startswith('Amazon'):
        short_name = short_name[7:]
    if short_name.startswith('AWS'):
        short_name = short_name[4:]
    if short_name and short_name.lower() not in official_name.lower():
        official_name += ' ({0})'.format(short_name)

    docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name))

    docs += '.. contents:: Table of Contents\n\n'

    docs += document_client(service_name, official_name, service_model)

    filename = (os.path.dirname(__file__) + '/data/resources/'
                '{0}-{1}.resources.json').format(service_name,
                                                 service_model.api_version)
    # We can't use a set here because dicts aren't hashable!
    models = {}
    if os.path.exists(filename):
        data = json.load(open(filename))
        model = ResourceModel(service_name, data['service'], data['resources'])

        for collection_model in model.collections:
            collection_model.parent_name = model.name
            models[collection_model.name] = {
                'type': 'collection',
                'model': collection_model
            }

        docs += document_resource(service_name, official_name, model,
                                  service_model)

        # First, collect all the models...
        for name, model in sorted(data['resources'].items(),
                                  key=lambda i:i[0]):
            resource_model = ResourceModel(name, model, data['resources'])
            if name not in models:
                models[name] = {'type': 'resource', 'model': resource_model}

            for collection_model in resource_model.collections:
                collection_model.parent_name = xform_name(resource_model.name)

                cname = collection_model.name + 'CollectionManager'
                if cname not in models:
                    models[cname] = {'type': 'collection',
                                     'model': collection_model}

        # Then render them out in alphabetical order.
        for name, item in sorted(models.items()):
            model = item['model']
            if item['type'] == 'resource':
                docs += document_resource(service_name, official_name,
                                          model, service_model)
            elif item['type'] == 'collection':
                docs += document_collection(
                    service_name, official_name, model,
                    model.resource.model, service_model)

    return docs