class ServiceWaiterFunctionalTest(BaseEnvVar):
"""
This class is used as a base class if you want to functionally test the
waiters for a specific service.
"""
def setUp(self):
super(ServiceWaiterFunctionalTest, self).setUp()
self.data_path = os.path.join(
os.path.dirname(botocore.__file__), 'data')
self.environ['BOTO_DATA_PATH'] = self.data_path
self.loader = Loader(self.data_path)
def get_waiter_model(self, service, api_version=None):
"""Get the waiter model for the service."""
return WaiterModel(
self.loader.load_service_model(
service, type_name='waiters-2', api_version=api_version))
def get_service_model(self, service, api_version=None):
"""Get the service model for the service."""
return ServiceModel(
self.loader.load_service_model(
service, type_name='service-2', api_version=api_version),
service_name=service)
def get_resources(self) -> List[Resource]:
boto_loader = Loader()
if self.options.services:
aws_services = self.options.services
else:
aws_services = boto_loader.list_available_services(
type_name="service-2")
resources = []
allowed_actions = self.get_policies_allowed_actions()
if self.options.verbose:
message_handler(
"Analyzing listing operations across {} service...".format(
len(aws_services)),
"HEADER",
)
with ThreadPoolExecutor(PARALLEL_SERVICE_CALLS) as executor:
results = executor.map(
lambda aws_service: self.analyze_service(
aws_service, boto_loader, allowed_actions),
aws_services,
)
for service_resources in results:
if service_resources is not None:
resources.extend(service_resources)
return resources
class ServiceWaiterFunctionalTest(BaseEnvVar):
"""
This class is used as a base class if you want to functionally test the
waiters for a specific service.
"""
def setUp(self):
super(ServiceWaiterFunctionalTest, self).setUp()
self.data_path = os.path.join(os.path.dirname(botocore.__file__),
'data')
self.environ['AWS_DATA_PATH'] = self.data_path
self.loader = Loader([self.data_path])
def get_waiter_model(self, service, api_version=None):
"""Get the waiter model for the service."""
with mock.patch('botocore.loaders.Loader.list_available_services',
return_value=[service]):
return WaiterModel(
self.loader.load_service_model(service,
type_name='waiters-2',
api_version=api_version))
def get_service_model(self, service, api_version=None):
"""Get the service model for the service."""
with mock.patch('botocore.loaders.Loader.list_available_services',
return_value=[service]):
return ServiceModel(self.loader.load_service_model(
service, type_name='service-2', api_version=api_version),
service_name=service)
def test_search_path_on_windows(self):
# On windows, the search path is separated by ';' chars.
self.environ['BOTO_DATA_PATH'] = 'c:\\path1;c:\\path2'
# The builtin botocore data path is added as the last element
# so we're only interested in checking the two that we've added.
loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
paths = loader.get_search_paths()[:-1]
self.assertEqual(paths, ['c:\\path1', 'c:\\path2'])
def setUp(self):
super(LoaderTestCase, self).setUp()
self.data_path = os.path.join(os.path.dirname(__file__), 'data')
self.environ['BOTO_DATA_PATH'] = self.data_path
self.loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
# Make sure the cache is clear.
self.loader._cache.clear()
def test_search_path_on_windows(self):
# On windows, the search path is separated by ';' chars.
self.environ['BOTO_DATA_PATH'] = 'c:\\path1;c:\\path2'
# The builtin botocore data path is added as the last element
# so we're only interested in checking the two that we've added.
loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
paths = loader.get_search_paths()[:-1]
self.assertEqual(paths, ['c:\\path1', 'c:\\path2'])
def test_data_not_found_raises_exception(self):
class FakeLoader(object):
def load_file(self, name):
# Returning None indicates that the
# loader couldn't find anything.
return None
loader = Loader(file_loader=FakeLoader())
with self.assertRaises(DataNotFoundError):
loader.load_data('baz')
def test_data_not_found_raises_exception(self):
class FakeLoader(object):
def load_file(self, name):
# Returning None indicates that the
# loader couldn't find anything.
return None
loader = Loader(file_loader=FakeLoader())
with self.assertRaises(DataNotFoundError):
loader.load_data('baz')
def test_load_service_model(self):
class FakeLoader(object):
def load_file(self, name):
return ['loaded data']
loader = Loader(extra_search_paths=['foo'],
file_loader=FakeLoader(),
include_default_search_paths=False)
loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
loaded = loader.load_service_model('baz', type_name='service-2')
self.assertEqual(loaded, ['loaded data'])
def test_load_data_uses_loader(self):
search_paths = ['foo', 'bar', 'baz']
class FakeLoader(object):
def load_file(self, name):
expected_ending = os.path.join('bar', 'baz')
if name.endswith(expected_ending):
return ['loaded data']
loader = Loader(extra_search_paths=search_paths,
file_loader=FakeLoader())
loaded = loader.load_data('baz')
self.assertEqual(loaded, ['loaded data'])
def test_load_data_uses_loader(self):
search_paths = ['foo', 'bar', 'baz']
class FakeLoader(object):
def load_file(self, name):
expected_ending = os.path.join('bar', 'baz')
if name.endswith(expected_ending):
return ['loaded data']
loader = Loader(extra_search_paths=search_paths,
file_loader=FakeLoader())
loaded = loader.load_data('baz')
self.assertEqual(loaded, ['loaded data'])
def test_load_service_model_enforces_case(self):
class FakeLoader(object):
def load_file(self, name):
return ['loaded data']
loader = Loader(extra_search_paths=['foo'],
file_loader=FakeLoader(),
include_default_search_paths=False)
loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
loader.list_available_services = mock.Mock(return_value=['baz'])
with self.assertRaises(ValidationError):
loader.load_service_model('BAZ', type_name='service-2')
def setUp(self):
super(TestMergeExtras, self).setUp()
self.file_loader = mock.Mock()
self.data_loader = Loader(extra_search_paths=['datapath'],
file_loader=self.file_loader,
include_default_search_paths=False)
self.data_loader.determine_latest_version = mock.Mock(
return_value='2015-03-01')
self.data_loader.list_available_services = mock.Mock(
return_value=['myservice'])
isdir_mock = mock.Mock(return_value=True)
self.isdir_patch = mock.patch('os.path.isdir', isdir_mock)
self.isdir_patch.start()
def test_load_service_model_uses_provided_type_name(self):
loader = Loader(extra_search_paths=['foo'],
file_loader=mock.Mock(),
include_default_search_paths=False)
loader.list_available_services = mock.Mock(return_value=['baz'])
# Should have a) the unknown service name and b) list of valid
# service names.
provided_type_name = 'not-service-2'
with self.assertRaisesRegex(UnknownServiceError,
'Unknown service.*BAZ.*baz'):
loader.load_service_model('BAZ', type_name=provided_type_name)
loader.list_available_services.assert_called_with(provided_type_name)
def test_load_service_model_uses_provided_type_name(self):
loader = Loader(extra_search_paths=['foo'],
file_loader=mock.Mock(),
include_default_search_paths=False)
loader.list_available_services = mock.Mock(return_value=['baz'])
# Should have a) the unknown service name and b) list of valid
# service names.
provided_type_name = 'not-service-2'
with self.assertRaisesRegexp(UnknownServiceError,
'Unknown service.*BAZ.*baz'):
loader.load_service_model(
'BAZ', type_name=provided_type_name)
loader.list_available_services.assert_called_with(provided_type_name)
def test_load_data_overridden(self):
self.overrides_path = os.path.join(os.path.dirname(__file__),
'data_overrides')
self.environ['BOTO_DATA_PATH'] = "{0}{1}{2}".format(
self.overrides_path, os.pathsep, self.data_path)
loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
# This should load the data the first data it finds.
data = loader.load_service_model('someservice',
api_version='2012-10-01')
# An overridden key.
self.assertEqual(data['api_version'], '2012-10-01')
# A key unique to the base.
self.assertEqual(data['something-else'], 'another')
# Ensure a key present in other variants is not there.
self.assertTrue('Purpose' not in data)
def setUp(self):
self.root_dir = tempfile.mkdtemp()
self.version_dirs = os.path.join(self.root_dir, 'myservice',
'2014-01-01')
os.makedirs(self.version_dirs)
self.model_file = os.path.join(self.version_dirs, 'service-2.json')
self.waiter_model_file = os.path.join(self.version_dirs,
'waiters-2.json')
self.paginator_model_file = os.path.join(self.version_dirs,
'paginators-1.json')
self.resource_model_file = os.path.join(self.version_dirs,
'resources-1.json')
self.example_model_file = os.path.join(self.version_dirs,
'examples-1.json')
self.json_model = {}
self.waiter_json_model = {}
self.paginator_json_model = {}
self.resource_json_model = {}
self._setup_models()
self._write_models()
self.doc_name = 'MyDoc'
self.doc_structure = DocumentStructure(self.doc_name)
self.loader = Loader(extra_search_paths=[self.root_dir])
self.botocore_session = botocore.session.get_session()
self.botocore_session.register_component('data_loader', self.loader)
self.session = Session(botocore_session=self.botocore_session,
region_name='us-east-1')
self.client = self.session.client('myservice', 'us-east-1')
self.resource = self.session.resource('myservice', 'us-east-1')
def setup_client(self):
with open(self.example_model_file, 'w') as f:
json.dump(self.example_json_model, f)
with open(self.waiter_model_file, 'w') as f:
json.dump(self.waiter_json_model, f)
with open(self.paginator_model_file, 'w') as f:
json.dump(self.paginator_json_model, f)
with open(self.model_file, 'w') as f:
json.dump(self.json_model, f)
self.loader = Loader(extra_search_paths=[self.root_dir])
endpoint_resolver = mock.Mock()
endpoint_resolver.construct_endpoint.return_value = {
'hostname': 'foo.us-east-1',
'partition': 'aws',
'endpointName': 'us-east-1',
'signatureVersions': ['v4']
}
self.creator = ClientCreator(loader=self.loader,
endpoint_resolver=endpoint_resolver,
user_agent='user-agent',
event_emitter=self.events,
retry_handler_factory=mock.Mock(),
retry_config_translator=mock.Mock())
self.client = self.creator.create_client('myservice', 'us-east-1')
def setup_client(self):
with open(self.waiter_model_file, 'w') as f:
json.dump(self.waiter_json_model, f)
with open(self.paginator_model_file, 'w') as f:
json.dump(self.paginator_json_model, f)
with open(self.model_file, 'w') as f:
json.dump(self.json_model, f)
self.loader = Loader(extra_search_paths=[self.root_dir])
endpoint_resolver = mock.Mock()
endpoint_resolver.construct_endpoint.return_value = {
'properties': {},
'uri': 'http://foo'
}
self.creator = ClientCreator(
loader=self.loader, endpoint_resolver=endpoint_resolver,
user_agent='user-agent', event_emitter=self.events,
retry_handler_factory=mock.Mock(),
retry_config_translator=mock.Mock())
self.client = self.creator.create_client('myservice', 'us-east-1')
def test_can_initialize_with_search_paths(self):
loader = Loader(extra_search_paths=['foo', 'bar'])
# Note that the extra search paths are before
# the customer/builtin data paths.
self.assertEqual(loader.search_paths, [
'foo', 'bar', loader.CUSTOMER_DATA_PATH, loader.BUILTIN_DATA_PATH
])
def test_load_service_model_enforces_case(self):
class FakeLoader(object):
def load_file(self, name):
return ['loaded data']
loader = Loader(extra_search_paths=['foo'],
file_loader=FakeLoader(),
include_default_search_paths=False)
loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
loader.list_available_services = mock.Mock(return_value=['baz'])
# Should have a) the unknown service name and b) list of valid
# service names.
with self.assertRaisesRegexp(UnknownServiceError,
'Unknown service.*BAZ.*baz'):
loader.load_service_model('BAZ', type_name='service-2')
def setUp(self):
super(LoaderTestCase, self).setUp()
self.data_path = os.path.join(os.path.dirname(__file__), 'data')
self.environ['BOTO_DATA_PATH'] = self.data_path
self.loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
# Make sure the cache is clear.
self.loader._cache.clear()
def _test_model_is_not_lost(service_name, type_name,
previous_version, latest_version):
# Make sure if a paginator and/or waiter exists in previous version,
# there will be a successor existing in latest version.
loader = Loader()
try:
previous = loader.load_service_model(
service_name, type_name, previous_version)
except DataNotFoundError:
pass
else:
try:
latest = loader.load_service_model(
service_name, type_name, latest_version)
except DataNotFoundError as e:
raise AssertionError(
"%s must exist for %s: %s" % (type_name, service_name, e))
def test_include_default_extras(self):
self.data_loader = Loader(extra_search_paths=['datapath'],
file_loader=self.file_loader,
include_default_search_paths=False,
include_default_extras=False)
self.data_loader.determine_latest_version = mock.Mock(
return_value='2015-03-01')
self.data_loader.list_available_services = mock.Mock(
return_value=['myservice'])
service_data = {'foo': 'service', 'bar': 'service'}
service_data_copy = copy.copy(service_data)
sdk_extras = {'merge': {'foo': 'sdk'}}
self.file_loader.load_file.side_effect = [service_data, sdk_extras]
loaded = self.data_loader.load_service_model('myservice', 'service-2')
self.assertEqual(loaded, service_data_copy)
def test_can_initialize_with_search_paths(self):
loader = Loader(extra_search_paths=['foo', 'bar'])
self.assertIn('foo', loader.search_paths)
self.assertIn('bar', loader.search_paths)
# We should also always add the default search
# paths even if the loader is initialized with
# additional search paths.
self.assertEqual(len(loader.search_paths), 4)
def test_paginators_and_waiters_are_not_lost_in_new_version(
service_name, type_name, previous_version, latest_version):
# Make sure if a paginator and/or waiter exists in previous version,
# there will be a successor existing in latest version.
loader = Loader()
try:
previous = loader.load_service_model(service_name, type_name,
previous_version)
except DataNotFoundError:
pass
else:
try:
latest = loader.load_service_model(service_name, type_name,
latest_version)
except DataNotFoundError as e:
raise AssertionError(
f"{type_name} must exist for {service_name}: {e}")
def _get_data_loader():
# Creates a botocore data loader that loads custom data files
# FIRST, creating a precedence for custom files.
data_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)),
BOTOCORE_DATA_FOLDER_NAME)
return Loader(extra_search_paths=[data_folder, Loader.BUILTIN_DATA_PATH],
include_default_search_paths=False)
def setup_client_and_resource(self):
self._write_models()
self.loader = Loader(extra_search_paths=[self.root_dir])
self.botocore_session = botocore.session.get_session()
self.botocore_session.register_component('data_loader', self.loader)
self.session = Session(
botocore_session=self.botocore_session, region_name='us-east-1')
self.client = self.session.client('myservice', 'us-east-1')
self.resource = self.session.resource('myservice', 'us-east-1')
def loader_with_fake_dirs(self):
mock_file_loader = mock.Mock()
mock_file_loader.exists = self.fake_exists
search_paths = list(self.fake_directories)
loader = Loader(extra_search_paths=search_paths,
include_default_search_paths=False,
file_loader=mock_file_loader)
with mock.patch('os.listdir', self.fake_listdir):
with mock.patch('os.path.isdir', mock.Mock(return_value=True)):
yield loader
def test_default_search_paths(self):
loader = Loader()
self.assertEqual(len(loader.search_paths), 2)
# We should also have ~/.aws/models added to
# the search path. To deal with cross platform
# issues we'll just check for a path that ends
# with .aws/models.
home_dir_path = os.path.join('.aws', 'models')
self.assertTrue(
any(p.endswith(home_dir_path) for p in loader.search_paths))
def setUp(self):
self.session = get_session()
self.files = FileCreator()
# Create our own loader for the unit test and not rely on the
# customer's actual ~/.aws/models nor the builtin botocore data
# directory.
self.customer_data_root = os.path.join(self.files.rootdir, 'customer')
os.mkdir(self.customer_data_root)
self.builtin_data_root = os.path.join(self.files.rootdir, 'builtin')
os.mkdir(self.builtin_data_root)
self.data_loader = Loader(
[self.customer_data_root, self.builtin_data_root],
include_default_search_paths=False)
self.data_loader.CUSTOMER_DATA_PATH = self.customer_data_root
self.session.register_component('data_loader', self.data_loader)
# Since we are using a custom data loader, we need to remove
# retries since these try to get loaded when the service model
# is loaded.
self.session.unregister('service-data-loaded',
register_retries_for_service)
# Add some models into the builtin model directory
# We are going to add two models. One with a matching service name
# and endpoint and another without.
self.matching_service = 'matching'
self.non_matching_service = 'nonmatching'
self.non_matching_prefix = 'nonmatching-prefix'
self.default_api_version = '2015-10-01'
matching_service_path = os.path.join(self.builtin_data_root,
self.matching_service,
self.default_api_version,
'service-2.json')
os.makedirs(os.path.dirname(matching_service_path))
non_matching_service_path = os.path.join(self.builtin_data_root,
self.non_matching_service,
self.default_api_version,
'service-2.json')
os.makedirs(os.path.dirname(non_matching_service_path))
# Write the models to the builtin directory
with open(matching_service_path, 'w') as f:
json.dump(
self._create_service_definition(self.matching_service,
self.default_api_version), f)
with open(non_matching_service_path, 'w') as f:
json.dump(
self._create_service_definition(self.non_matching_prefix,
self.default_api_version), f)
def _add_doctype_service_model(file_creator, session, model=None):
if model is None:
model = DOCTYPE_MODEL
file_creator.create_file(
os.path.join('doctype', '2011-06-15', 'service-2.json'),
json.dumps(model)
)
data_path = session.get_config_variable('data_path').split(os.pathsep)
loader = Loader()
loader.search_paths.extend(data_path + [file_creator.rootdir])
session.register_component('data_loader', loader)
def __init__(self,
session_vars=None,
event_hooks=None,
include_builtin_handlers=True,
loader=None):
"""
Create a new Session object.
:type session_vars: dict
:param session_vars: A dictionary that is used to override some or all
of the environment variables associated with this session. The
key/value pairs defined in this dictionary will override the
corresponding variables defined in ``SessionVariables``.
:type event_hooks: BaseEventHooks
:param event_hooks: The event hooks object to use. If one is not
provided, an event hooks object will be automatically created
for you.
:type include_builtin_handlers: bool
:param include_builtin_handlers: Indicates whether or not to
automatically register builtin handlers.
"""
self.session_var_map = copy.copy(self.SessionVariables)
if session_vars:
self.session_var_map.update(session_vars)
if event_hooks is None:
self._events = HierarchicalEmitter()
else:
self._events = event_hooks
if include_builtin_handlers:
self._register_builtin_handlers(self._events)
self.user_agent_name = 'Botocore'
self.user_agent_version = __version__
self.user_agent_extra = ''
self._profile = None
self._config = None
self._credentials = None
self._profile_map = None
self._provider = None
# This is a dict that stores per session specific config variable
# overrides via set_config_variable().
self._session_instance_vars = {}
if loader is None:
loader = Loader()
self._loader = loader
# _data_paths_added is used to track whether or not we added
# extra paths to the loader. We will do this lazily
# only when we ask for the loader.
self._data_paths_added = False
self._components = ComponentLocator()
self._register_components()
def test_load_data_overridden(self):
self.overrides_path = os.path.join(
os.path.dirname(__file__),
'data_overrides'
)
self.environ['BOTO_DATA_PATH'] = "{0}{1}{2}".format(
self.overrides_path,
os.pathsep,
self.data_path
)
loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
# This should load the data the first data it finds.
data = loader.load_service_model(
'someservice',
api_version='2012-10-01'
)
# An overridden key.
self.assertEqual(data['api_version'], '2012-10-01')
# A key unique to the base.
self.assertEqual(data['something-else'], 'another')
# Ensure a key present in other variants is not there.
self.assertTrue('Purpose' not in data)
def setUp(self):
super(TestMergeExtras, self).setUp()
self.file_loader = mock.Mock()
self.data_loader = Loader(
extra_search_paths=['datapath'], file_loader=self.file_loader,
include_default_search_paths=False)
self.data_loader.determine_latest_version = mock.Mock(
return_value='2015-03-01')
self.data_loader.list_available_services = mock.Mock(
return_value=['myservice'])
isdir_mock = mock.Mock(return_value=True)
self.isdir_patch = mock.patch('os.path.isdir', isdir_mock)
self.isdir_patch.start()
class ServiceWaiterFunctionalTest(BaseEnvVar):
"""
This class is used as a base class if you want to functionally test the
waiters for a specific service.
"""
def setUp(self):
super(ServiceWaiterFunctionalTest, self).setUp()
self.data_path = os.path.join(os.path.dirname(botocore.__file__),
'data')
self.environ['BOTO_DATA_PATH'] = self.data_path
self.loader = Loader(self.data_path)
def get_waiter_model(self, service, api_version=None):
"""
Get the waiter model for the service
"""
service = os.path.join('aws', service)
model_version = self.loader.determine_latest(service, api_version)
# Some wierd formatting required to get the name of the model
# correct. Right now this is returned: YYYY-MM-DD.api
# We need: YYYY-MM-DD
model_version = ''.join(model_version.split('.')[:-1])
waiter_model = model_version + '.waiters'
return WaiterModel(self.loader.load_data(waiter_model))
class ServiceWaiterFunctionalTest(BaseEnvVar):
"""
This class is used as a base class if you want to functionally test the
waiters for a specific service.
"""
def setUp(self):
super(ServiceWaiterFunctionalTest, self).setUp()
self.data_path = os.path.join(
os.path.dirname(botocore.__file__), 'data')
self.environ['BOTO_DATA_PATH'] = self.data_path
self.loader = Loader(self.data_path)
def get_waiter_model(self, service, api_version=None):
"""
Get the waiter model for the service
"""
service = os.path.join('aws', service)
model_version = self.loader.determine_latest(service, api_version)
# Some wierd formatting required to get the name of the model
# correct. Right now this is returned: YYYY-MM-DD.api
# We need: YYYY-MM-DD
model_version = ''.join(model_version.split('.')[:-1])
waiter_model = model_version + '.waiters'
return WaiterModel(self.loader.load_data(waiter_model))
def test_include_default_extras(self):
self.data_loader = Loader(
extra_search_paths=['datapath'], file_loader=self.file_loader,
include_default_search_paths=False,
include_default_extras=False)
self.data_loader.determine_latest_version = mock.Mock(
return_value='2015-03-01')
self.data_loader.list_available_services = mock.Mock(
return_value=['myservice'])
service_data = {'foo': 'service', 'bar': 'service'}
service_data_copy = copy.copy(service_data)
sdk_extras = {'merge': {'foo': 'sdk'}}
self.file_loader.load_file.side_effect = [service_data, sdk_extras]
loaded = self.data_loader.load_service_model('myservice', 'service-2')
self.assertEqual(loaded, service_data_copy)
class ServiceWaiterFunctionalTest(BaseEnvVar):
"""
This class is used as a base class if you want to functionally test the
waiters for a specific service.
"""
def setUp(self):
super(ServiceWaiterFunctionalTest, self).setUp()
self.data_path = os.path.join(os.path.dirname(botocore.__file__),
'data')
self.environ['BOTO_DATA_PATH'] = self.data_path
self.loader = Loader(self.data_path)
def get_waiter_model(self, service, api_version=None):
"""Get the waiter model for the service."""
return WaiterModel(
self.loader.load_service_model(service,
type_name='waiters-2',
api_version=api_version))
def test_load_service_model_enforces_case(self):
class FakeLoader(object):
def load_file(self, name):
return ['loaded data']
loader = Loader(extra_search_paths=['foo'],
file_loader=FakeLoader(),
include_default_search_paths=False)
loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
loader.list_available_services = mock.Mock(return_value=['baz'])
with self.assertRaises(ValidationError):
loader.load_service_model('BAZ', type_name='service-2')
def test_load_service_model_enforces_case(self):
class FakeLoader(object):
def load_file(self, name):
return ['loaded data']
loader = Loader(extra_search_paths=['foo'],
file_loader=FakeLoader(),
include_default_search_paths=False)
loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
loader.list_available_services = mock.Mock(return_value=['baz'])
# Should have a) the unknown service name and b) list of valid
# service names.
with self.assertRaisesRegex(UnknownServiceError,
'Unknown service.*BAZ.*baz'):
loader.load_service_model('BAZ', type_name='service-2')
class TestMergeExtras(BaseEnvVar):
def setUp(self):
super(TestMergeExtras, self).setUp()
self.file_loader = mock.Mock()
self.data_loader = Loader(
extra_search_paths=['datapath'], file_loader=self.file_loader,
include_default_search_paths=False)
self.data_loader.determine_latest_version = mock.Mock(
return_value='2015-03-01')
self.data_loader.list_available_services = mock.Mock(
return_value=['myservice'])
isdir_mock = mock.Mock(return_value=True)
self.isdir_patch = mock.patch('os.path.isdir', isdir_mock)
self.isdir_patch.start()
def tearDown(self):
super(TestMergeExtras, self).tearDown()
self.isdir_patch.stop()
def test_merge_extras(self):
service_data = {'foo': 'service', 'bar': 'service'}
sdk_extras = {'merge': {'foo': 'sdk'}}
self.file_loader.load_file.side_effect = [service_data, sdk_extras]
loaded = self.data_loader.load_service_model('myservice', 'service-2')
expected = {'foo': 'sdk', 'bar': 'service'}
self.assertEqual(loaded, expected)
call_args = self.file_loader.load_file.call_args_list
call_args = [c[0][0] for c in call_args]
base_path = os.path.join('datapath', 'myservice', '2015-03-01')
expected_call_args = [
os.path.join(base_path, 'service-2'),
os.path.join(base_path, 'service-2.sdk-extras')
]
self.assertEqual(call_args, expected_call_args)
def test_extras_not_found(self):
service_data = {'foo': 'service', 'bar': 'service'}
service_data_copy = copy.copy(service_data)
self.file_loader.load_file.side_effect = [service_data, None]
loaded = self.data_loader.load_service_model('myservice', 'service-2')
self.assertEqual(loaded, service_data_copy)
def test_no_merge_in_extras(self):
service_data = {'foo': 'service', 'bar': 'service'}
service_data_copy = copy.copy(service_data)
self.file_loader.load_file.side_effect = [service_data, {}]
loaded = self.data_loader.load_service_model('myservice', 'service-2')
self.assertEqual(loaded, service_data_copy)
def test_include_default_extras(self):
self.data_loader = Loader(
extra_search_paths=['datapath'], file_loader=self.file_loader,
include_default_search_paths=False,
include_default_extras=False)
self.data_loader.determine_latest_version = mock.Mock(
return_value='2015-03-01')
self.data_loader.list_available_services = mock.Mock(
return_value=['myservice'])
service_data = {'foo': 'service', 'bar': 'service'}
service_data_copy = copy.copy(service_data)
sdk_extras = {'merge': {'foo': 'sdk'}}
self.file_loader.load_file.side_effect = [service_data, sdk_extras]
loaded = self.data_loader.load_service_model('myservice', 'service-2')
self.assertEqual(loaded, service_data_copy)
def test_append_extra_type(self):
service_data = {'foo': 'service', 'bar': 'service'}
sdk_extras = {'merge': {'foo': 'sdk'}}
cli_extras = {'merge': {'cli': True}}
self.file_loader.load_file.side_effect = [
service_data, sdk_extras, cli_extras]
self.data_loader.extras_types.append('cli')
loaded = self.data_loader.load_service_model('myservice', 'service-2')
expected = {'foo': 'sdk', 'bar': 'service', 'cli': True}
self.assertEqual(loaded, expected)
call_args = self.file_loader.load_file.call_args_list
call_args = [c[0][0] for c in call_args]
base_path = os.path.join('datapath', 'myservice', '2015-03-01')
expected_call_args = [
os.path.join(base_path, 'service-2'),
os.path.join(base_path, 'service-2.sdk-extras'),
os.path.join(base_path, 'service-2.cli-extras')
]
self.assertEqual(call_args, expected_call_args)
def test_sdk_empty_extras_skipped(self):
service_data = {'foo': 'service', 'bar': 'service'}
cli_extras = {'merge': {'foo': 'cli'}}
self.file_loader.load_file.side_effect = [
service_data, None, cli_extras]
self.data_loader.extras_types.append('cli')
loaded = self.data_loader.load_service_model('myservice', 'service-2')
expected = {'foo': 'cli', 'bar': 'service'}
self.assertEqual(loaded, expected)
from botocore.loaders import Loader
#Make named tuple for operations:
#'http', 'name', 'documentation', 'output', 'input'
#Make named tuple for shapes:
#'required', 'members', 'member', 'type', 'min', 'enum', 'max', 'documentation', 'pattern'
#Mane named tuple for shapes.members:
#'locationName', 'shape', 'queryName', 'idempotencyToken', 'documentation'
l = Loader()
m = l.load_service_model('ec2', 'service-2')
kset = set()
def process_operations(v1):
'''v1 is a collection of operations entities for the service.
Inputs and outputs for c2s are regular, except:
{"CancelConversionTask,OrderedDict([('shape', 'CancelConversionRequest')])"}
'''
for k2, v2 in v1.items():
print(f'\n{k2}')
for k3, v3 in v2.items():
print(f'\n\t{k3}')
if k3 == 'http':
#OrderedDict([('method', 'POST'), ('requestUri', '/')])
#print(f'\n\t\thttp is {v3}')
#if v3["method"] != 'POST' or v3["requestUri"] != '/':
from botocore.exceptions import ReadTimeoutError
from botocore.loaders import Loader
from botocore.model import ServiceModel
from awswrangler import _utils, exceptions
from awswrangler._config import apply_configs
from awswrangler.s3._describe import size_objects
_logger: logging.Logger = logging.getLogger(__name__)
_S3_RETRYABLE_ERRORS: Tuple[Any, Any, Any] = (socket.timeout, ConnectionError, ReadTimeoutError)
_MIN_WRITE_BLOCK: int = 5_242_880 # 5 MB (5 * 2**20)
_MIN_PARALLEL_READ_BLOCK: int = 5_242_880 # 5 MB (5 * 2**20)
_BOTOCORE_LOADER = Loader()
_S3_JSON_MODEL = _BOTOCORE_LOADER.load_service_model(service_name="s3", type_name="service-2")
_S3_SERVICE_MODEL = ServiceModel(_S3_JSON_MODEL, service_name="s3")
def _snake_to_camel_case(s: str) -> str:
return "".join(c.title() for c in s.split("_"))
def get_botocore_valid_kwargs(function_name: str, s3_additional_kwargs: Dict[str, Any]) -> Dict[str, Any]:
"""Filter and keep only the valid botocore key arguments."""
s3_operation_model = _S3_SERVICE_MODEL.operation_model(_snake_to_camel_case(function_name))
allowed_kwargs = s3_operation_model.input_shape.members.keys() # pylint: disable=E1101
return {k: v for k, v in s3_additional_kwargs.items() if k in allowed_kwargs}
def setUp(self):
super(ServiceWaiterFunctionalTest, self).setUp()
self.data_path = os.path.join(
os.path.dirname(botocore.__file__), 'data')
self.environ['BOTO_DATA_PATH'] = self.data_path
self.loader = Loader(self.data_path)
def test_data_path_not_required(self):
loader = Loader()
self.assertEqual(loader.data_path, '')
loader.data_path = 'foo:bar'
self.assertEqual(loader.data_path, 'foo:bar')
class LoaderTestCase(BaseEnvVar):
def setUp(self):
super(LoaderTestCase, self).setUp()
self.data_path = os.path.join(os.path.dirname(__file__), 'data')
self.environ['BOTO_DATA_PATH'] = self.data_path
self.loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
# Make sure the cache is clear.
self.loader._cache.clear()
def test_data_path_not_required(self):
loader = Loader()
self.assertEqual(loader.data_path, '')
loader.data_path = 'foo:bar'
self.assertEqual(loader.data_path, 'foo:bar')
def test_get_search_paths(self):
paths = self.loader.get_search_paths()
self.assertTrue(self.data_path in paths)
def test_determine_latest_no_version(self):
path = self.loader.determine_latest('someservice')
self.assertEqual(path, os.path.join('someservice',
'2013-08-21.normal'))
def test_determine_latest_with_version(self):
path = self.loader.determine_latest(
'someservice',
api_version='2012-10-01'
)
self.assertEqual(path, os.path.join('someservice',
'2012-10-01.normal'))
def test_determine_latest_with_version_the_wrong_way(self):
with self.assertRaises(ApiVersionNotFoundError):
self.loader.determine_latest('someservice/2012-10-01')
def test_determine_latest_with_version_not_found(self):
with self.assertRaises(ApiVersionNotFoundError):
path = self.loader.determine_latest(
'someservice',
api_version='2010-02-02'
)
def test_load_data_plain_file(self):
data = self.loader.load_data('foo')
self.assertEqual(data['test_key_1'], 'test_value_1')
def test_load_data_plain_file_nonexistant(self):
with self.assertRaises(DataNotFoundError):
data = self.loader.load_data('i_totally_dont_exist')
def test_load_service_model_latest_without_version(self):
data = self.loader.load_service_model('someservice')
self.assertEqual(data['api_version'], '2013-08-21')
def test_load_service_model_with_version(self):
data = self.loader.load_service_model(
'someservice',
api_version='2012-10-01'
)
self.assertEqual(data['api_version'], '2012-10-01')
def test_load_service_model_version_not_found(self):
with self.assertRaises(ApiVersionNotFoundError):
data = self.loader.load_service_model(
'someservice',
api_version='2010-02-02'
)
def test_load_service_model_data_path_order(self):
# There's an s3/ directory both in our custom BOTO_DATA_PATH
# directory as well as in the botocore/data/ directory.
# Our path should win since the default built in path is always
# last.
data = self.loader.load_service_model('aws/s3')
self.assertTrue(data.get('WAS_OVERRIDEN_VIA_DATA_PATH'),
"S3 model was loaded from botocore's default "
"data path instead of from the BOTO_DATA_PATH"
" directory.")
def test_list_available_services(self):
avail = self.loader.list_available_services('')
self.assertEqual(sorted(avail), [
'aws',
'aws',
'someservice',
'sub',
])
aws_avail = self.loader.list_available_services('aws')
self.assertTrue(len(aws_avail) > 10)
self.assertTrue('ec2' in aws_avail)
def test_load_data_overridden(self):
self.overrides_path = os.path.join(
os.path.dirname(__file__),
'data_overrides'
)
self.environ['BOTO_DATA_PATH'] = "{0}{1}{2}".format(
self.overrides_path,
os.pathsep,
self.data_path
)
loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
# This should load the data the first data it finds.
data = loader.load_service_model(
'someservice',
api_version='2012-10-01'
)
# An overridden key.
self.assertEqual(data['api_version'], '2012-10-01')
# A key unique to the base.
self.assertEqual(data['something-else'], 'another')
# Ensure a key present in other variants is not there.
self.assertTrue('Purpose' not in data)
@mock.patch('os.pathsep', ';')
def test_search_path_on_windows(self):
# On windows, the search path is separated by ';' chars.
self.environ['BOTO_DATA_PATH'] = 'c:\\path1;c:\\path2'
# The builtin botocore data path is added as the last element
# so we're only interested in checking the two that we've added.
loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
paths = loader.get_search_paths()[:-1]
self.assertEqual(paths, ['c:\\path1', 'c:\\path2'])
def test_error_raised_if_service_does_not_exist(self):
loader = Loader(extra_search_paths=[],
include_default_search_paths=False)
with self.assertRaises(DataNotFoundError):
loader.determine_latest_version('unknownservice', 'service-2')