class ServiceWaiterFunctionalTest(BaseEnvVar):
"""
This class is used as a base class if you want to functionally test the
waiters for a specific service.
"""
def setUp(self):
super(ServiceWaiterFunctionalTest, self).setUp()
self.data_path = os.path.join(
os.path.dirname(botocore.__file__), 'data')
self.environ['BOTO_DATA_PATH'] = self.data_path
self.loader = Loader(self.data_path)
def get_waiter_model(self, service, api_version=None):
"""Get the waiter model for the service."""
return WaiterModel(
self.loader.load_service_model(
service, type_name='waiters-2', api_version=api_version))
def get_service_model(self, service, api_version=None):
"""Get the service model for the service."""
return ServiceModel(
self.loader.load_service_model(
service, type_name='service-2', api_version=api_version),
service_name=service)
class ServiceWaiterFunctionalTest(BaseEnvVar):
"""
This class is used as a base class if you want to functionally test the
waiters for a specific service.
"""
def setUp(self):
super(ServiceWaiterFunctionalTest, self).setUp()
self.data_path = os.path.join(os.path.dirname(botocore.__file__),
'data')
self.environ['AWS_DATA_PATH'] = self.data_path
self.loader = Loader([self.data_path])
def get_waiter_model(self, service, api_version=None):
"""Get the waiter model for the service."""
with mock.patch('botocore.loaders.Loader.list_available_services',
return_value=[service]):
return WaiterModel(
self.loader.load_service_model(service,
type_name='waiters-2',
api_version=api_version))
def get_service_model(self, service, api_version=None):
"""Get the service model for the service."""
with mock.patch('botocore.loaders.Loader.list_available_services',
return_value=[service]):
return ServiceModel(self.loader.load_service_model(
service, type_name='service-2', api_version=api_version),
service_name=service)
def test_load_service_model_enforces_case(self):
class FakeLoader(object):
def load_file(self, name):
return ['loaded data']
loader = Loader(extra_search_paths=['foo'],
file_loader=FakeLoader(),
include_default_search_paths=False)
loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
loader.list_available_services = mock.Mock(return_value=['baz'])
with self.assertRaises(ValidationError):
loader.load_service_model('BAZ', type_name='service-2')
def test_load_service_model_enforces_case(self):
class FakeLoader(object):
def load_file(self, name):
return ['loaded data']
loader = Loader(extra_search_paths=['foo'],
file_loader=FakeLoader(),
include_default_search_paths=False)
loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
loader.list_available_services = mock.Mock(return_value=['baz'])
with self.assertRaises(ValidationError):
loader.load_service_model('BAZ', type_name='service-2')
def test_load_service_model_uses_provided_type_name(self):
loader = Loader(extra_search_paths=['foo'],
file_loader=mock.Mock(),
include_default_search_paths=False)
loader.list_available_services = mock.Mock(return_value=['baz'])
# Should have a) the unknown service name and b) list of valid
# service names.
provided_type_name = 'not-service-2'
with self.assertRaisesRegex(UnknownServiceError,
'Unknown service.*BAZ.*baz'):
loader.load_service_model('BAZ', type_name=provided_type_name)
loader.list_available_services.assert_called_with(provided_type_name)
def test_paginators_and_waiters_are_not_lost_in_new_version(
service_name, type_name, previous_version, latest_version):
# Make sure if a paginator and/or waiter exists in previous version,
# there will be a successor existing in latest version.
loader = Loader()
try:
loader.load_service_model(service_name, type_name, previous_version)
except DataNotFoundError:
pass
else:
try:
loader.load_service_model(service_name, type_name, latest_version)
except DataNotFoundError as e:
raise AssertionError(
f"{type_name} must exist for {service_name}: {e}")
def test_load_service_model_uses_provided_type_name(self):
loader = Loader(extra_search_paths=['foo'],
file_loader=mock.Mock(),
include_default_search_paths=False)
loader.list_available_services = mock.Mock(return_value=['baz'])
# Should have a) the unknown service name and b) list of valid
# service names.
provided_type_name = 'not-service-2'
with self.assertRaisesRegexp(UnknownServiceError,
'Unknown service.*BAZ.*baz'):
loader.load_service_model(
'BAZ', type_name=provided_type_name)
loader.list_available_services.assert_called_with(provided_type_name)
def test_load_service_model_enforces_case(self):
class FakeLoader(object):
def load_file(self, name):
return ['loaded data']
loader = Loader(extra_search_paths=['foo'],
file_loader=FakeLoader(),
include_default_search_paths=False)
loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
loader.list_available_services = mock.Mock(return_value=['baz'])
# Should have a) the unknown service name and b) list of valid
# service names.
with self.assertRaisesRegex(UnknownServiceError,
'Unknown service.*BAZ.*baz'):
loader.load_service_model('BAZ', type_name='service-2')
def test_load_service_model_enforces_case(self):
class FakeLoader(object):
def load_file(self, name):
return ['loaded data']
loader = Loader(extra_search_paths=['foo'],
file_loader=FakeLoader(),
include_default_search_paths=False)
loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
loader.list_available_services = mock.Mock(return_value=['baz'])
# Should have a) the unknown service name and b) list of valid
# service names.
with self.assertRaisesRegexp(UnknownServiceError,
'Unknown service.*BAZ.*baz'):
loader.load_service_model('BAZ', type_name='service-2')
def _test_model_is_not_lost(service_name, type_name,
previous_version, latest_version):
# Make sure if a paginator and/or waiter exists in previous version,
# there will be a successor existing in latest version.
loader = Loader()
try:
previous = loader.load_service_model(
service_name, type_name, previous_version)
except DataNotFoundError:
pass
else:
try:
latest = loader.load_service_model(
service_name, type_name, latest_version)
except DataNotFoundError as e:
raise AssertionError(
"%s must exist for %s: %s" % (type_name, service_name, e))
def test_load_service_model(self):
class FakeLoader(object):
def load_file(self, name):
return ['loaded data']
loader = Loader(extra_search_paths=['foo'],
file_loader=FakeLoader(),
include_default_search_paths=False)
loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
loaded = loader.load_service_model('baz', type_name='service-2')
self.assertEqual(loaded, ['loaded data'])
def test_load_data_overridden(self):
self.overrides_path = os.path.join(os.path.dirname(__file__),
'data_overrides')
self.environ['BOTO_DATA_PATH'] = "{0}{1}{2}".format(
self.overrides_path, os.pathsep, self.data_path)
loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
# This should load the data the first data it finds.
data = loader.load_service_model('someservice',
api_version='2012-10-01')
# An overridden key.
self.assertEqual(data['api_version'], '2012-10-01')
# A key unique to the base.
self.assertEqual(data['something-else'], 'another')
# Ensure a key present in other variants is not there.
self.assertTrue('Purpose' not in data)
class ServiceWaiterFunctionalTest(BaseEnvVar):
"""
This class is used as a base class if you want to functionally test the
waiters for a specific service.
"""
def setUp(self):
super(ServiceWaiterFunctionalTest, self).setUp()
self.data_path = os.path.join(os.path.dirname(botocore.__file__),
'data')
self.environ['BOTO_DATA_PATH'] = self.data_path
self.loader = Loader(self.data_path)
def get_waiter_model(self, service, api_version=None):
"""Get the waiter model for the service."""
return WaiterModel(
self.loader.load_service_model(service,
type_name='waiters-2',
api_version=api_version))
def test_load_data_overridden(self):
self.overrides_path = os.path.join(
os.path.dirname(__file__),
'data_overrides'
)
self.environ['BOTO_DATA_PATH'] = "{0}{1}{2}".format(
self.overrides_path,
os.pathsep,
self.data_path
)
loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
# This should load the data the first data it finds.
data = loader.load_service_model(
'someservice',
api_version='2012-10-01'
)
# An overridden key.
self.assertEqual(data['api_version'], '2012-10-01')
# A key unique to the base.
self.assertEqual(data['something-else'], 'another')
# Ensure a key present in other variants is not there.
self.assertTrue('Purpose' not in data)
class LoaderTestCase(BaseEnvVar):
def setUp(self):
super(LoaderTestCase, self).setUp()
self.data_path = os.path.join(os.path.dirname(__file__), 'data')
self.environ['BOTO_DATA_PATH'] = self.data_path
self.loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
# Make sure the cache is clear.
self.loader._cache.clear()
def test_data_path_not_required(self):
loader = Loader()
self.assertEqual(loader.data_path, '')
loader.data_path = 'foo:bar'
self.assertEqual(loader.data_path, 'foo:bar')
def test_get_search_paths(self):
paths = self.loader.get_search_paths()
self.assertTrue(self.data_path in paths)
def test_determine_latest_no_version(self):
path = self.loader.determine_latest('someservice')
self.assertEqual(path, os.path.join('someservice',
'2013-08-21.normal'))
def test_determine_latest_with_version(self):
path = self.loader.determine_latest(
'someservice',
api_version='2012-10-01'
)
self.assertEqual(path, os.path.join('someservice',
'2012-10-01.normal'))
def test_determine_latest_with_version_the_wrong_way(self):
with self.assertRaises(ApiVersionNotFoundError):
self.loader.determine_latest('someservice/2012-10-01')
def test_determine_latest_with_version_not_found(self):
with self.assertRaises(ApiVersionNotFoundError):
path = self.loader.determine_latest(
'someservice',
api_version='2010-02-02'
)
def test_load_data_plain_file(self):
data = self.loader.load_data('foo')
self.assertEqual(data['test_key_1'], 'test_value_1')
def test_load_data_plain_file_nonexistant(self):
with self.assertRaises(DataNotFoundError):
data = self.loader.load_data('i_totally_dont_exist')
def test_load_service_model_latest_without_version(self):
data = self.loader.load_service_model('someservice')
self.assertEqual(data['api_version'], '2013-08-21')
def test_load_service_model_with_version(self):
data = self.loader.load_service_model(
'someservice',
api_version='2012-10-01'
)
self.assertEqual(data['api_version'], '2012-10-01')
def test_load_service_model_version_not_found(self):
with self.assertRaises(ApiVersionNotFoundError):
data = self.loader.load_service_model(
'someservice',
api_version='2010-02-02'
)
def test_load_service_model_data_path_order(self):
# There's an s3/ directory both in our custom BOTO_DATA_PATH
# directory as well as in the botocore/data/ directory.
# Our path should win since the default built in path is always
# last.
data = self.loader.load_service_model('aws/s3')
self.assertTrue(data.get('WAS_OVERRIDEN_VIA_DATA_PATH'),
"S3 model was loaded from botocore's default "
"data path instead of from the BOTO_DATA_PATH"
" directory.")
def test_list_available_services(self):
avail = self.loader.list_available_services('')
self.assertEqual(sorted(avail), [
'aws',
'aws',
'someservice',
'sub',
])
aws_avail = self.loader.list_available_services('aws')
self.assertTrue(len(aws_avail) > 10)
self.assertTrue('ec2' in aws_avail)
def test_load_data_overridden(self):
self.overrides_path = os.path.join(
os.path.dirname(__file__),
'data_overrides'
)
self.environ['BOTO_DATA_PATH'] = "{0}{1}{2}".format(
self.overrides_path,
os.pathsep,
self.data_path
)
loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
# This should load the data the first data it finds.
data = loader.load_service_model(
'someservice',
api_version='2012-10-01'
)
# An overridden key.
self.assertEqual(data['api_version'], '2012-10-01')
# A key unique to the base.
self.assertEqual(data['something-else'], 'another')
# Ensure a key present in other variants is not there.
self.assertTrue('Purpose' not in data)
@mock.patch('os.pathsep', ';')
def test_search_path_on_windows(self):
# On windows, the search path is separated by ';' chars.
self.environ['BOTO_DATA_PATH'] = 'c:\\path1;c:\\path2'
# The builtin botocore data path is added as the last element
# so we're only interested in checking the two that we've added.
loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
paths = loader.get_search_paths()[:-1]
self.assertEqual(paths, ['c:\\path1', 'c:\\path2'])
class TestMergeExtras(BaseEnvVar):
def setUp(self):
super(TestMergeExtras, self).setUp()
self.file_loader = mock.Mock()
self.data_loader = Loader(
extra_search_paths=['datapath'], file_loader=self.file_loader,
include_default_search_paths=False)
self.data_loader.determine_latest_version = mock.Mock(
return_value='2015-03-01')
self.data_loader.list_available_services = mock.Mock(
return_value=['myservice'])
isdir_mock = mock.Mock(return_value=True)
self.isdir_patch = mock.patch('os.path.isdir', isdir_mock)
self.isdir_patch.start()
def tearDown(self):
super(TestMergeExtras, self).tearDown()
self.isdir_patch.stop()
def test_merge_extras(self):
service_data = {'foo': 'service', 'bar': 'service'}
sdk_extras = {'merge': {'foo': 'sdk'}}
self.file_loader.load_file.side_effect = [service_data, sdk_extras]
loaded = self.data_loader.load_service_model('myservice', 'service-2')
expected = {'foo': 'sdk', 'bar': 'service'}
self.assertEqual(loaded, expected)
call_args = self.file_loader.load_file.call_args_list
call_args = [c[0][0] for c in call_args]
base_path = os.path.join('datapath', 'myservice', '2015-03-01')
expected_call_args = [
os.path.join(base_path, 'service-2'),
os.path.join(base_path, 'service-2.sdk-extras')
]
self.assertEqual(call_args, expected_call_args)
def test_extras_not_found(self):
service_data = {'foo': 'service', 'bar': 'service'}
service_data_copy = copy.copy(service_data)
self.file_loader.load_file.side_effect = [service_data, None]
loaded = self.data_loader.load_service_model('myservice', 'service-2')
self.assertEqual(loaded, service_data_copy)
def test_no_merge_in_extras(self):
service_data = {'foo': 'service', 'bar': 'service'}
service_data_copy = copy.copy(service_data)
self.file_loader.load_file.side_effect = [service_data, {}]
loaded = self.data_loader.load_service_model('myservice', 'service-2')
self.assertEqual(loaded, service_data_copy)
def test_include_default_extras(self):
self.data_loader = Loader(
extra_search_paths=['datapath'], file_loader=self.file_loader,
include_default_search_paths=False,
include_default_extras=False)
self.data_loader.determine_latest_version = mock.Mock(
return_value='2015-03-01')
self.data_loader.list_available_services = mock.Mock(
return_value=['myservice'])
service_data = {'foo': 'service', 'bar': 'service'}
service_data_copy = copy.copy(service_data)
sdk_extras = {'merge': {'foo': 'sdk'}}
self.file_loader.load_file.side_effect = [service_data, sdk_extras]
loaded = self.data_loader.load_service_model('myservice', 'service-2')
self.assertEqual(loaded, service_data_copy)
def test_append_extra_type(self):
service_data = {'foo': 'service', 'bar': 'service'}
sdk_extras = {'merge': {'foo': 'sdk'}}
cli_extras = {'merge': {'cli': True}}
self.file_loader.load_file.side_effect = [
service_data, sdk_extras, cli_extras]
self.data_loader.extras_types.append('cli')
loaded = self.data_loader.load_service_model('myservice', 'service-2')
expected = {'foo': 'sdk', 'bar': 'service', 'cli': True}
self.assertEqual(loaded, expected)
call_args = self.file_loader.load_file.call_args_list
call_args = [c[0][0] for c in call_args]
base_path = os.path.join('datapath', 'myservice', '2015-03-01')
expected_call_args = [
os.path.join(base_path, 'service-2'),
os.path.join(base_path, 'service-2.sdk-extras'),
os.path.join(base_path, 'service-2.cli-extras')
]
self.assertEqual(call_args, expected_call_args)
def test_sdk_empty_extras_skipped(self):
service_data = {'foo': 'service', 'bar': 'service'}
cli_extras = {'merge': {'foo': 'cli'}}
self.file_loader.load_file.side_effect = [
service_data, None, cli_extras]
self.data_loader.extras_types.append('cli')
loaded = self.data_loader.load_service_model('myservice', 'service-2')
expected = {'foo': 'cli', 'bar': 'service'}
self.assertEqual(loaded, expected)
from botocore.loaders import Loader
from botocore.model import ServiceModel
from awswrangler import _utils, exceptions
from awswrangler._config import apply_configs
from awswrangler.s3._describe import size_objects
_logger: logging.Logger = logging.getLogger(__name__)
_S3_RETRYABLE_ERRORS: Tuple[Any, Any, Any] = (socket.timeout, ConnectionError, ReadTimeoutError)
_MIN_WRITE_BLOCK: int = 5_242_880 # 5 MB (5 * 2**20)
_MIN_PARALLEL_READ_BLOCK: int = 5_242_880 # 5 MB (5 * 2**20)
_BOTOCORE_LOADER = Loader()
_S3_JSON_MODEL = _BOTOCORE_LOADER.load_service_model(service_name="s3", type_name="service-2")
_S3_SERVICE_MODEL = ServiceModel(_S3_JSON_MODEL, service_name="s3")
def _snake_to_camel_case(s: str) -> str:
return "".join(c.title() for c in s.split("_"))
def get_botocore_valid_kwargs(function_name: str, s3_additional_kwargs: Dict[str, Any]) -> Dict[str, Any]:
"""Filter and keep only the valid botocore key arguments."""
s3_operation_model = _S3_SERVICE_MODEL.operation_model(_snake_to_camel_case(function_name))
allowed_kwargs = s3_operation_model.input_shape.members.keys() # pylint: disable=E1101
return {k: v for k, v in s3_additional_kwargs.items() if k in allowed_kwargs}
def _fetch_range(
class LoaderTestCase(BaseEnvVar):
def setUp(self):
super(LoaderTestCase, self).setUp()
self.data_path = os.path.join(os.path.dirname(__file__), 'data')
self.environ['BOTO_DATA_PATH'] = self.data_path
self.loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
# Make sure the cache is clear.
self.loader._cache.clear()
def test_data_path_not_required(self):
loader = Loader()
self.assertEqual(loader.data_path, '')
loader.data_path = 'foo:bar'
self.assertEqual(loader.data_path, 'foo:bar')
def test_get_search_paths(self):
paths = self.loader.get_search_paths()
self.assertTrue(self.data_path in paths)
def test_determine_latest_no_version(self):
path = self.loader.determine_latest('someservice')
self.assertEqual(path, os.path.join('someservice', '2013-08-21.api'))
def test_determine_latest_with_version(self):
path = self.loader.determine_latest('someservice',
api_version='2012-10-01')
self.assertEqual(path, os.path.join('someservice', '2012-10-01.api'))
def test_determine_latest_with_version_the_wrong_way(self):
with self.assertRaises(ApiVersionNotFoundError):
self.loader.determine_latest('someservice/2012-10-01')
def test_determine_latest_with_version_not_found(self):
with self.assertRaises(ApiVersionNotFoundError):
path = self.loader.determine_latest('someservice',
api_version='2010-02-02')
def test_load_data_plain_file(self):
data = self.loader.load_data('foo')
self.assertEqual(data['test_key_1'], 'test_value_1')
def test_load_data_plain_file_nonexistant(self):
with self.assertRaises(DataNotFoundError):
data = self.loader.load_data('i_totally_dont_exist')
def test_load_service_model_latest_without_version(self):
data = self.loader.load_service_model('someservice')
self.assertEqual(data['api_version'], '2013-08-21')
def test_load_service_model_with_version(self):
data = self.loader.load_service_model('someservice',
api_version='2012-10-01')
self.assertEqual(data['api_version'], '2012-10-01')
def test_load_service_model_version_not_found(self):
with self.assertRaises(ApiVersionNotFoundError):
data = self.loader.load_service_model('someservice',
api_version='2010-02-02')
def test_load_service_model_data_path_order(self):
# There's an s3/ directory both in our custom BOTO_DATA_PATH
# directory as well as in the botocore/data/ directory.
# Our path should win since the default built in path is always
# last.
data = self.loader.load_service_model('aws/s3')
self.assertTrue(
data.get('WAS_OVERRIDEN_VIA_DATA_PATH'),
"S3 model was loaded from botocore's default "
"data path instead of from the BOTO_DATA_PATH"
" directory.")
def test_list_available_services(self):
avail = self.loader.list_available_services('')
self.assertEqual(sorted(avail), [
'aws',
'aws',
'someservice',
'sub',
])
aws_avail = self.loader.list_available_services('aws')
self.assertTrue(len(aws_avail) > 10)
self.assertTrue('ec2' in aws_avail)
def test_load_data_overridden(self):
self.overrides_path = os.path.join(os.path.dirname(__file__),
'data_overrides')
self.environ['BOTO_DATA_PATH'] = "{0}{1}{2}".format(
self.overrides_path, os.pathsep, self.data_path)
loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
# This should load the data the first data it finds.
data = loader.load_service_model('someservice',
api_version='2012-10-01')
# An overridden key.
self.assertEqual(data['api_version'], '2012-10-01')
# A key unique to the base.
self.assertEqual(data['something-else'], 'another')
# Ensure a key present in other variants is not there.
self.assertTrue('Purpose' not in data)
@mock.patch('os.pathsep', ';')
def test_search_path_on_windows(self):
# On windows, the search path is separated by ';' chars.
self.environ['BOTO_DATA_PATH'] = 'c:\\path1;c:\\path2'
# The builtin botocore data path is added as the last element
# so we're only interested in checking the two that we've added.
loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
paths = loader.get_search_paths()[:-1]
self.assertEqual(paths, ['c:\\path1', 'c:\\path2'])
class TestMergeExtras(BaseEnvVar):
def setUp(self):
super(TestMergeExtras, self).setUp()
self.file_loader = mock.Mock()
self.data_loader = Loader(extra_search_paths=['datapath'],
file_loader=self.file_loader,
include_default_search_paths=False)
self.data_loader.determine_latest_version = mock.Mock(
return_value='2015-03-01')
self.data_loader.list_available_services = mock.Mock(
return_value=['myservice'])
isdir_mock = mock.Mock(return_value=True)
self.isdir_patch = mock.patch('os.path.isdir', isdir_mock)
self.isdir_patch.start()
def tearDown(self):
super(TestMergeExtras, self).tearDown()
self.isdir_patch.stop()
def test_merge_extras(self):
service_data = {'foo': 'service', 'bar': 'service'}
sdk_extras = {'merge': {'foo': 'sdk'}}
self.file_loader.load_file.side_effect = [service_data, sdk_extras]
loaded = self.data_loader.load_service_model('myservice', 'service-2')
expected = {'foo': 'sdk', 'bar': 'service'}
self.assertEqual(loaded, expected)
call_args = self.file_loader.load_file.call_args_list
call_args = [c[0][0] for c in call_args]
base_path = os.path.join('datapath', 'myservice', '2015-03-01')
expected_call_args = [
os.path.join(base_path, 'service-2'),
os.path.join(base_path, 'service-2.sdk-extras')
]
self.assertEqual(call_args, expected_call_args)
def test_extras_not_found(self):
service_data = {'foo': 'service', 'bar': 'service'}
service_data_copy = copy.copy(service_data)
self.file_loader.load_file.side_effect = [service_data, None]
loaded = self.data_loader.load_service_model('myservice', 'service-2')
self.assertEqual(loaded, service_data_copy)
def test_no_merge_in_extras(self):
service_data = {'foo': 'service', 'bar': 'service'}
service_data_copy = copy.copy(service_data)
self.file_loader.load_file.side_effect = [service_data, {}]
loaded = self.data_loader.load_service_model('myservice', 'service-2')
self.assertEqual(loaded, service_data_copy)
def test_include_default_extras(self):
self.data_loader = Loader(extra_search_paths=['datapath'],
file_loader=self.file_loader,
include_default_search_paths=False,
include_default_extras=False)
self.data_loader.determine_latest_version = mock.Mock(
return_value='2015-03-01')
self.data_loader.list_available_services = mock.Mock(
return_value=['myservice'])
service_data = {'foo': 'service', 'bar': 'service'}
service_data_copy = copy.copy(service_data)
sdk_extras = {'merge': {'foo': 'sdk'}}
self.file_loader.load_file.side_effect = [service_data, sdk_extras]
loaded = self.data_loader.load_service_model('myservice', 'service-2')
self.assertEqual(loaded, service_data_copy)
def test_append_extra_type(self):
service_data = {'foo': 'service', 'bar': 'service'}
sdk_extras = {'merge': {'foo': 'sdk'}}
cli_extras = {'merge': {'cli': True}}
self.file_loader.load_file.side_effect = [
service_data, sdk_extras, cli_extras
]
self.data_loader.extras_types.append('cli')
loaded = self.data_loader.load_service_model('myservice', 'service-2')
expected = {'foo': 'sdk', 'bar': 'service', 'cli': True}
self.assertEqual(loaded, expected)
call_args = self.file_loader.load_file.call_args_list
call_args = [c[0][0] for c in call_args]
base_path = os.path.join('datapath', 'myservice', '2015-03-01')
expected_call_args = [
os.path.join(base_path, 'service-2'),
os.path.join(base_path, 'service-2.sdk-extras'),
os.path.join(base_path, 'service-2.cli-extras')
]
self.assertEqual(call_args, expected_call_args)
def test_sdk_empty_extras_skipped(self):
service_data = {'foo': 'service', 'bar': 'service'}
cli_extras = {'merge': {'foo': 'cli'}}
self.file_loader.load_file.side_effect = [
service_data, None, cli_extras
]
self.data_loader.extras_types.append('cli')
loaded = self.data_loader.load_service_model('myservice', 'service-2')
expected = {'foo': 'cli', 'bar': 'service'}
self.assertEqual(loaded, expected)
def handle_method(fragment):
if fragment["Type"] != "AWS::ApiGateway::Method":
response_string = "Macro only supports \"AWS::ApiGateway::Method\", user supplied {}"
raise InvalidTypeException(response_string.format(fragment["Type"]))
service_name = fragment["Properties"]["Integration"].pop("Service").lower()
action = fragment["Properties"]["Integration"].pop("Action")
response_maps = fragment["Properties"]["Integration"].pop("ResponseMaps")
try:
fragment.pop("Fn::Transform")
except:
pass
loader = Loader()
service_description = loader.load_service_model(service_name=service_name, type_name='service-2')
service_model = ServiceModel(service_description)
protocol = service_model.protocol
op_model = service_model.operation_model(action["Name"])
request_parameters = action.get("Parameters", {})
params = dict(ChainMap(*request_parameters))
print("params: {}".format(params))
serializer = create_serializer(protocol)
response_parser = create_parser(protocol)
print(service_model.protocol)
request = serializer.serialize_to_request(params, op_model)
request_object = AWSRequest(
method=request['method'],
url=get_endpoint(service_model.service_name),
data=request['body'],
headers=request['headers'])
X = request_object.prepare()
print("Raw request: {}".format(request))
print("Prepared request: {}".format(X))
integration = fragment["Properties"]["Integration"]
new_integration = integration_template()
# Copy the existing values to the new template
for entry in integration.keys():
new_integration[entry] = integration[entry]
# Add headers to cfn template
if X.headers is not None and callable(getattr(X.headers, "keys", None)):
for header in X.headers.keys():
if header.lower() != 'Content-Length'.lower():
new_integration["RequestParameters"].update({"integration.request.header.{}".format(header): "'{}'".format(X.headers[header])})
# Add Query Strings to cfn template
if 'query_string' in request and callable(getattr(request['query_string'], "keys", None)):
for query in request['query_string'].keys():
new_integration["RequestParameters"].update({"integration.request.querystring.{}".format(query): "'{}'".format(request['query_string'][query])})
# Set the body
if isinstance(X.body, str):
new_integration["RequestTemplates"]["application/json"] = X.body
else:
new_integration["RequestTemplates"]["application/json"] = str(X.body, "utf-8") if X.body else ''
new_integration["Uri"] = ":".join([
"arn",
"aws",
"apigateway",
REGION,
service_model.endpoint_prefix,
"path/" + request["url_path"]
])
new_integration["IntegrationHttpMethod"] = X.method
fragment["Properties"]["Integration"] = new_integration
print(fragment)
return fragment
from botocore.loaders import Loader
#Make named tuple for operations:
#'http', 'name', 'documentation', 'output', 'input'
#Make named tuple for shapes:
#'required', 'members', 'member', 'type', 'min', 'enum', 'max', 'documentation', 'pattern'
#Mane named tuple for shapes.members:
#'locationName', 'shape', 'queryName', 'idempotencyToken', 'documentation'
l = Loader()
m = l.load_service_model('ec2', 'service-2')
kset = set()
def process_operations(v1):
'''v1 is a collection of operations entities for the service.
Inputs and outputs for c2s are regular, except:
{"CancelConversionTask,OrderedDict([('shape', 'CancelConversionRequest')])"}
'''
for k2, v2 in v1.items():
print(f'\n{k2}')
for k3, v3 in v2.items():
print(f'\n\t{k3}')
if k3 == 'http':
#OrderedDict([('method', 'POST'), ('requestUri', '/')])
#print(f'\n\t\thttp is {v3}')
#if v3["method"] != 'POST' or v3["requestUri"] != '/':
def iam_file():
l = Loader()
# print(l.list_available_services('service-2'))
return l.load_service_model("iam", "service-2")
