def test_json_errors_parsing():
# The outputs/ directory has sample output responses
# For each file in outputs/ there's a corresponding file
# in expected/ that has the expected parsed response.
base_dir = os.path.join(os.path.dirname(__file__), 'json')
json_responses_dir = os.path.join(base_dir, 'errors')
expected_parsed_dir = os.path.join(base_dir, 'expected')
session = botocore.session.get_session()
for json_response_file in os.listdir(json_responses_dir):
# Files look like: 'datapipeline-create-pipeline.json'
service_name, operation_name = os.path.splitext(
json_response_file)[0].split('-', 1)
expected_parsed_response = os.path.join(expected_parsed_dir,
json_response_file)
raw_response_file = os.path.join(json_responses_dir,
json_response_file)
with open(expected_parsed_response) as f:
expected = json.load(f)
service_model = session.get_service_model(service_name)
operation_names = service_model.operation_names
operation_model = None
for op_name in operation_names:
if xform_name(op_name) == operation_name.replace('-', '_'):
operation_model = service_model.operation_model(op_name)
with open(raw_response_file, 'rb') as f:
raw_response_body = f.read()
yield _test_parsed_response, raw_response_file, \
raw_response_body, operation_model, expected
def test_valid(self):
session = botocore.session.get_session()
for impl in common.SimpleDescribe.__subclasses__():
if issubclass(impl, self.ignore):
continue
if getattr(impl, 'describe_action', None) is None:
continue
# API use must be versioned
assert len(impl.api_version or '') == 10, '{} is not versioned'.format(impl)
# Not allowed to set Describe.describe_filters AND implement Describe.get_describe_filters
is_df_set = bool(impl.describe_filters is not None)
is_gdf_set = 'get_describe_filters' in impl.__dict__
assert (is_gdf_set and not is_df_set) or not is_gdf_set
service = session.get_service_model(impl.service_name, impl.api_version)
methods = {xform_name(s): s for s in service.operation_names}
operation = service.operation_model(methods[impl.describe_action])
if impl.describe_envelope in ['@', '[@]']:
continue
if impl.describe_envelope.startswith('['):
continue
if '.' not in impl.describe_envelope and ':' not in impl.describe_envelope:
assert impl.describe_envelope in operation.output_shape.members
def test_json_errors_parsing():
# The outputs/ directory has sample output responses
# For each file in outputs/ there's a corresponding file
# in expected/ that has the expected parsed response.
base_dir = os.path.join(os.path.dirname(__file__), 'json')
json_responses_dir = os.path.join(base_dir, 'errors')
expected_parsed_dir = os.path.join(base_dir, 'expected')
session = botocore.session.get_session()
for json_response_file in os.listdir(json_responses_dir):
# Files look like: 'datapipeline-create-pipeline.json'
service_name, operation_name = os.path.splitext(
json_response_file)[0].split('-', 1)
expected_parsed_response = os.path.join(expected_parsed_dir,
json_response_file)
raw_response_file = os.path.join(json_responses_dir,
json_response_file)
with open(expected_parsed_response) as f:
expected = json.load(f)
service_model = session.get_service_model(service_name)
operation_names = service_model.operation_names
operation_model = None
for op_name in operation_names:
if xform_name(op_name) == operation_name.replace('-', '_'):
operation_model = service_model.operation_model(op_name)
with open(raw_response_file, 'rb') as f:
raw_response_body = f.read()
yield _test_parsed_response, raw_response_file, \
raw_response_body, operation_model, expected
def _all_inputs():
session = botocore.session.get_session()
for service_name in session.get_available_services():
service_model = session.get_service_model(service_name)
for operation_name in service_model.operation_names:
operation_model = service_model.operation_model(operation_name)
input_shape = operation_model.input_shape
if input_shape is not None and input_shape.members:
yield input_shape, service_name, operation_name
def _pagination_configs():
session = botocore.session.get_session()
loader = session.get_component('data_loader')
services = loader.list_available_services('paginators-1')
for service_name in services:
service_model = session.get_service_model(service_name)
page_config = loader.load_service_model(service_name, 'paginators-1',
service_model.api_version)
for op_name, single_config in page_config['pagination'].items():
yield (op_name, single_config, service_model)
def test_can_generate_all_inputs():
session = botocore.session.get_session()
generator = ArgumentGenerator()
for service_name in session.get_available_services():
service_model = session.get_service_model(service_name)
for operation_name in service_model.operation_names:
operation_model = service_model.operation_model(operation_name)
input_shape = operation_model.input_shape
if input_shape is not None and input_shape.members:
yield (_test_can_generate_skeleton, generator,
input_shape, service_name, operation_name)
def _shared_example_configs():
session = botocore.session.Session()
loader = session.get_component('data_loader')
services = loader.list_available_services('examples-1')
for service in services:
service_model = session.get_service_model(service)
example_config = loader.load_service_model(service, 'examples-1',
service_model.api_version)
examples = example_config.get("examples", {})
for operation, operation_examples in examples.items():
for example in operation_examples:
yield operation, example, service_model
def test_lint_shared_example_configs():
session = botocore.session.Session()
loader = session.get_component('data_loader')
services = loader.list_available_services('examples-1')
for service in services:
service_model = session.get_service_model(service)
example_config = loader.load_service_model(
service, 'examples-1', service_model.api_version
)
examples = example_config.get("examples", {})
for operation, operation_examples in examples.items():
for example in operation_examples:
yield _lint_single_example, operation, example, service_model
def test_lint_pagination_configs():
session = botocore.session.get_session()
loader = session.get_component('data_loader')
services = loader.list_available_services('paginators-1')
for service_name in services:
service_model = session.get_service_model(service_name)
page_config = loader.load_service_model(service_name,
'paginators-1',
service_model.api_version)
for op_name, single_config in page_config['pagination'].items():
yield (
_lint_single_paginator,
op_name,
single_config,
service_model
)
def _xml_test_cases():
for dp in ['responses', 'errors']:
data_path = os.path.join(os.path.dirname(__file__), 'xml')
data_path = os.path.join(data_path, dp)
session = create_session()
xml_files = glob.glob('%s/*.xml' % data_path)
service_names = set()
for fn in xml_files:
service_names.add(os.path.split(fn)[1].split('-')[0])
for service_name in service_names:
service_model = session.get_service_model(service_name)
service_xml_files = glob.glob('%s/%s-*.xml' %
(data_path, service_name))
for xmlfile in service_xml_files:
expected = _get_expected_parsed_result(xmlfile)
operation_model = _get_operation_model(service_model, xmlfile)
raw_response_body = _get_raw_response_body(xmlfile)
yield xmlfile, raw_response_body, operation_model, expected
def test_xml_parsing():
for dp in ['responses', 'errors']:
data_path = os.path.join(os.path.dirname(__file__), 'xml')
data_path = os.path.join(data_path, dp)
session = create_session()
xml_files = glob.glob('%s/*.xml' % data_path)
service_names = set()
for fn in xml_files:
service_names.add(os.path.split(fn)[1].split('-')[0])
for service_name in service_names:
service_model = session.get_service_model(service_name)
service_xml_files = glob.glob('%s/%s-*.xml' % (data_path,
service_name))
for xmlfile in service_xml_files:
expected = _get_expected_parsed_result(xmlfile)
operation_model = _get_operation_model(service_model, xmlfile)
raw_response_body = _get_raw_response_body(xmlfile)
yield _test_parsed_response, xmlfile, raw_response_body, \
operation_model, expected
def _uhg_test_json_parsing():
input_path = os.path.join(os.path.dirname(__file__), 'json')
input_path = os.path.join(input_path, 'inputs')
output_path = os.path.join(os.path.dirname(__file__), 'json')
output_path = os.path.join(output_path, 'outputs')
session = botocore.session.get_session()
jsonfiles = glob.glob('%s/*.json' % input_path)
service_names = set()
for fn in jsonfiles:
service_names.add(os.path.split(fn)[1].split('-')[0])
for service_name in service_names:
service_model = session.get_service_model(service_name)
service_json_files = glob.glob('%s/%s-*.json' % (input_path,
service_name))
for jsonfile in service_json_files:
expected = _get_expected_parsed_result(jsonfile)
operation_model = _get_operation_model(service_model, jsonfile)
with open(jsonfile, 'rb') as f:
raw_response_body = f.read()
yield _test_parsed_response, jsonfile, \
raw_response_body, operation_model, expected
def _uhg_test_json_parsing():
input_path = os.path.join(os.path.dirname(__file__), 'json')
input_path = os.path.join(input_path, 'inputs')
output_path = os.path.join(os.path.dirname(__file__), 'json')
output_path = os.path.join(output_path, 'outputs')
session = botocore.session.get_session()
jsonfiles = glob.glob('%s/*.json' % input_path)
service_names = set()
for fn in jsonfiles:
service_names.add(os.path.split(fn)[1].split('-')[0])
for service_name in service_names:
service_model = session.get_service_model(service_name)
service_json_files = glob.glob('%s/%s-*.json' %
(input_path, service_name))
for jsonfile in service_json_files:
expected = _get_expected_parsed_result(jsonfile)
operation_model = _get_operation_model(service_model, jsonfile)
with open(jsonfile, 'rb') as f:
raw_response_body = f.read()
yield _test_parsed_response, jsonfile, \
raw_response_body, operation_model, expected
def docs_for(service_name):
"""
Generate reference documentation (low and high level) for a service
by name. This generates docs for the latest available version.
:type service_name: string
:param service_name: The service short-name, like 'ec2'
:rtype: string
"""
session = botocore.session.get_session()
service_model = session.get_service_model(service_name)
print('Processing {0}-{1}'.format(service_name, service_model.api_version))
official_name = service_model.metadata.get(
'serviceAbbreviation', service_model.metadata.get('serviceFullName'))
docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name))
docs += '.. contents:: Table of Contents\n\n'
docs += document_client(service_name, official_name, service_model)
filename = (os.path.dirname(__file__) + '/data/resources/'
'{0}-{1}.resources.json').format(service_name,
service_model.api_version)
if os.path.exists(filename):
data = json.load(open(filename))
model = ResourceModel(service_name, data['service'], data['resources'])
docs += document_resource(service_name, official_name, model,
service_model)
for name, model in sorted(data['resources'].items(),
key=lambda i:i[0]):
resource_model = ResourceModel(name, model, data['resources'])
docs += document_resource(service_name, official_name,
resource_model, service_model)
return docs
def docs_for(service_name, session=None, resource_filename=None):
"""
Generate reference documentation (low and high level) for a service
by name. This generates docs for the latest available version.
:type service_name: string
:param service_name: The service short-name, like 'ec2'
:type session: botocore.session.Session
:param session: Existing pre-setup session or ``None``.
:type resource_filename: string
:param resource_filename: Full path to the service's resource JSON
description file. If unset, then this is
loaded from the default location.
:rtype: string
"""
if session is None:
session = botocore.session.get_session()
service_model = session.get_service_model(service_name)
print('Processing {0}-{1}'.format(service_name, service_model.api_version))
# The following creates an official name of 'Amazon Simple Storage
# Service (S3)' our of 'Amazon Simple Storage Service' and 'Amazon S3'.
# It tries to be smart, so for `Amazon DynamoDB' and 'DynamoDB' we would
# get an official name of 'Amazon DynamoDB'.
official_name = service_model.metadata.get('serviceFullName')
short_name = service_model.metadata.get('serviceAbbreviation', '')
if short_name.startswith('Amazon'):
short_name = short_name[7:]
if short_name.startswith('AWS'):
short_name = short_name[4:]
if short_name and short_name.lower() not in official_name.lower():
official_name += ' ({0})'.format(short_name)
docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name))
docs += '.. contents:: Table of Contents\n :depth: 2\n\n'
# TODO: Get this information from the model somehow in the future.
# For now creating and introspecing a client is a quick and
# dirty way to access waiters/paginators.
client = session.create_client(service_name,
aws_access_key_id='dummy',
aws_secret_access_key='dummy',
region_name='us-east-1')
docs += document_client(service_name, official_name, service_model, client)
docs += document_client_waiter(session, official_name, service_name,
service_model, client)
if resource_filename is None:
resource_filename = (os.path.dirname(__file__) + '/data/resources/'
'{0}-{1}.resources.json').format(
service_name, service_model.api_version)
# We can't use a set here because dicts aren't hashable!
models = {}
if os.path.exists(resource_filename):
data = json.load(open(resource_filename))
model = ResourceModel(service_name, data['service'], data['resources'])
for collection_model in model.collections:
collection_model.parent_name = model.name
models[collection_model.name] = {
'type': 'collection',
'model': collection_model
}
docs += document_resource(service_name, official_name, model,
service_model, session)
# First, collect all the models...
for name, model in sorted(data['resources'].items(),
key=lambda i: i[0]):
resource_model = ResourceModel(name, model, data['resources'])
shape = None
if resource_model.shape:
shape = service_model.shape_for(resource_model.shape)
resource_model.load_rename_map(shape)
if name not in models:
models[name] = {'type': 'resource', 'model': resource_model}
for collection_model in resource_model.collections:
collection_model.parent_name = xform_name(resource_model.name)
cname = collection_model.name + 'CollectionManager'
if cname not in models:
models[cname] = {
'type': 'collection',
'model': collection_model
}
# Then render them out in alphabetical order.
for name, item in sorted(models.items()):
model = item['model']
if item['type'] == 'resource':
docs += document_resource(service_name, official_name, model,
service_model, session)
elif item['type'] == 'collection':
docs += document_collection(service_name, official_name, model,
model.resource.model,
service_model)
return docs
def docs_for(service_name):
"""
Generate reference documentation (low and high level) for a service
by name. This generates docs for the latest available version.
:type service_name: string
:param service_name: The service short-name, like 'ec2'
:rtype: string
"""
session = botocore.session.get_session()
service_model = session.get_service_model(service_name)
print('Processing {0}-{1}'.format(service_name, service_model.api_version))
# The following creates an official name of 'Amazon Simple Storage
# Service (S3)' our of 'Amazon Simple Storage Service' and 'Amazon S3'.
# It tries to be smart, so for `Amazon DynamoDB' and 'DynamoDB' we would
# get an official name of 'Amazon DynamoDB'.
official_name = service_model.metadata.get('serviceFullName')
short_name = service_model.metadata.get('serviceAbbreviation', '')
if short_name.startswith('Amazon'):
short_name = short_name[7:]
if short_name.startswith('AWS'):
short_name = short_name[4:]
if short_name and short_name.lower() not in official_name.lower():
official_name += ' ({0})'.format(short_name)
docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name))
docs += '.. contents:: Table of Contents\n :depth: 2\n\n'
docs += document_client(service_name, official_name, service_model)
docs += document_client_waiter(session, official_name, service_name,
service_model)
filename = (os.path.dirname(__file__) + '/data/resources/'
'{0}-{1}.resources.json').format(service_name,
service_model.api_version)
# We can't use a set here because dicts aren't hashable!
models = {}
if os.path.exists(filename):
data = json.load(open(filename))
model = ResourceModel(service_name, data['service'], data['resources'])
for collection_model in model.collections:
collection_model.parent_name = model.name
models[collection_model.name] = {
'type': 'collection',
'model': collection_model
}
docs += document_resource(service_name, official_name, model,
service_model, session)
# First, collect all the models...
for name, model in sorted(data['resources'].items(),
key=lambda i: i[0]):
resource_model = ResourceModel(name, model, data['resources'])
shape = None
if resource_model.shape:
shape = service_model.shape_for(resource_model.shape)
resource_model.load_rename_map(shape)
if name not in models:
models[name] = {'type': 'resource', 'model': resource_model}
for collection_model in resource_model.collections:
collection_model.parent_name = xform_name(resource_model.name)
cname = collection_model.name + 'CollectionManager'
if cname not in models:
models[cname] = {
'type': 'collection',
'model': collection_model
}
# Then render them out in alphabetical order.
for name, item in sorted(models.items()):
model = item['model']
if item['type'] == 'resource':
docs += document_resource(service_name, official_name, model,
service_model, session)
elif item['type'] == 'collection':
docs += document_collection(service_name, official_name, model,
model.resource.model,
service_model)
return docs
def docs_for(service_name, session=None, resource_filename=None):
"""
Generate reference documentation (low and high level) for a service
by name. This generates docs for the latest available version.
:type service_name: string
:param service_name: The service short-name, like 'ec2'
:type session: botocore.session.Session
:param session: Existing pre-setup session or ``None``.
:type resource_filename: string
:param resource_filename: Full path to the service's resource JSON
description file. If unset, then this is
loaded from the default location.
:rtype: string
"""
if session is None:
session = botocore.session.get_session()
service_model = session.get_service_model(service_name)
print('Processing {0}-{1}'.format(service_name, service_model.api_version))
# The following creates an official name of 'Amazon Simple Storage
# Service (S3)' our of 'Amazon Simple Storage Service' and 'Amazon S3'.
# It tries to be smart, so for `Amazon DynamoDB' and 'DynamoDB' we would
# get an official name of 'Amazon DynamoDB'.
official_name = service_model.metadata.get('serviceFullName')
short_name = service_model.metadata.get('serviceAbbreviation', '')
if short_name.startswith('Amazon'):
short_name = short_name[7:]
if short_name.startswith('AWS'):
short_name = short_name[4:]
if short_name and short_name.lower() not in official_name.lower():
official_name += ' ({0})'.format(short_name)
docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name))
docs += '.. contents:: Table of Contents\n :depth: 2\n\n'
# TODO: Get this information from the model somehow in the future.
# For now creating and introspecing a client is a quick and
# dirty way to access waiters/paginators.
client = session.create_client(service_name, aws_access_key_id='dummy',
aws_secret_access_key='dummy',
region_name='us-east-1')
docs += document_client(service_name, official_name, service_model,
client)
docs += document_client_waiter(session, official_name, service_name,
service_model, client)
if resource_filename is None:
resource_filename = (os.path.dirname(__file__) + '/data/resources/'
'{0}-{1}.resources.json').format(
service_name, service_model.api_version)
# We can't use a set here because dicts aren't hashable!
models = {}
if os.path.exists(resource_filename):
data = json.load(open(resource_filename))
model = ResourceModel(service_name, data['service'], data['resources'])
for collection_model in model.collections:
collection_model.parent_name = model.name
models[collection_model.name] = {
'type': 'collection',
'model': collection_model
}
docs += document_resource(service_name, official_name, model,
service_model, session)
# First, collect all the models...
for name, model in sorted(data['resources'].items(),
key=lambda i:i[0]):
resource_model = ResourceModel(name, model, data['resources'])
shape = None
if resource_model.shape:
shape = service_model.shape_for(resource_model.shape)
resource_model.load_rename_map(shape)
if name not in models:
models[name] = {'type': 'resource', 'model': resource_model}
for collection_model in resource_model.collections:
collection_model.parent_name = xform_name(resource_model.name)
cname = collection_model.name + 'CollectionManager'
if cname not in models:
models[cname] = {'type': 'collection',
'model': collection_model}
# Then render them out in alphabetical order.
for name, item in sorted(models.items()):
model = item['model']
if item['type'] == 'resource':
docs += document_resource(service_name, official_name,
model, service_model, session)
elif item['type'] == 'collection':
docs += document_collection(
service_name, official_name, model,
model.resource.model, service_model)
return docs
def docs_for(service_name):
"""
Generate reference documentation (low and high level) for a service
by name. This generates docs for the latest available version.
:type service_name: string
:param service_name: The service short-name, like 'ec2'
:rtype: string
"""
session = botocore.session.get_session()
service_model = session.get_service_model(service_name)
print('Processing {0}-{1}'.format(service_name, service_model.api_version))
# The following creates an official name of 'Amazon Simple Storage
# Service (S3)' our of 'Amazon Simple Storage Service' and 'Amazon S3'.
# It tries to be smart, so for `Amazon DynamoDB' and 'DynamoDB' we would
# get an official name of 'Amazon DynamoDB'.
official_name = service_model.metadata.get('serviceFullName')
short_name = service_model.metadata.get('serviceAbbreviation', '')
if short_name.startswith('Amazon'):
short_name = short_name[7:]
if short_name.startswith('AWS'):
short_name = short_name[4:]
if short_name and short_name.lower() not in official_name.lower():
official_name += ' ({0})'.format(short_name)
docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name))
docs += '.. contents:: Table of Contents\n\n'
docs += document_client(service_name, official_name, service_model)
filename = (os.path.dirname(__file__) + '/data/resources/'
'{0}-{1}.resources.json').format(service_name,
service_model.api_version)
# We can't use a set here because dicts aren't hashable!
models = {}
if os.path.exists(filename):
data = json.load(open(filename))
model = ResourceModel(service_name, data['service'], data['resources'])
for collection_model in model.collections:
collection_model.parent_name = model.name
models[collection_model.name] = {
'type': 'collection',
'model': collection_model
}
docs += document_resource(service_name, official_name, model,
service_model)
# First, collect all the models...
for name, model in sorted(data['resources'].items(),
key=lambda i:i[0]):
resource_model = ResourceModel(name, model, data['resources'])
if name not in models:
models[name] = {'type': 'resource', 'model': resource_model}
for collection_model in resource_model.collections:
collection_model.parent_name = xform_name(resource_model.name)
cname = collection_model.name + 'CollectionManager'
if cname not in models:
models[cname] = {'type': 'collection',
'model': collection_model}
# Then render them out in alphabetical order.
for name, item in sorted(models.items()):
model = item['model']
if item['type'] == 'resource':
docs += document_resource(service_name, official_name,
model, service_model)
elif item['type'] == 'collection':
docs += document_collection(
service_name, official_name, model,
model.resource.model, service_model)
return docs
