Esempio n. 1
0
class ServiceWaiterFunctionalTest(BaseEnvVar):
    """
    This class is used as a base class if you want to functionally test the
    waiters for a specific service.
    """

    def setUp(self):
        super(ServiceWaiterFunctionalTest, self).setUp()
        self.data_path = os.path.join(
            os.path.dirname(botocore.__file__), 'data')
        self.environ['BOTO_DATA_PATH'] = self.data_path
        self.loader = Loader(self.data_path)

    def get_waiter_model(self, service, api_version=None):
        """Get the waiter model for the service."""
        return WaiterModel(
            self.loader.load_service_model(
                service, type_name='waiters-2', api_version=api_version))

    def get_service_model(self, service, api_version=None):
        """Get the service model for the service."""
        return ServiceModel(
            self.loader.load_service_model(
                service, type_name='service-2', api_version=api_version),
            service_name=service)
Esempio n. 2
0
    def get_resources(self) -> List[Resource]:
        boto_loader = Loader()
        if self.options.services:
            aws_services = self.options.services
        else:
            aws_services = boto_loader.list_available_services(
                type_name="service-2")
        resources = []
        allowed_actions = self.get_policies_allowed_actions()

        if self.options.verbose:
            message_handler(
                "Analyzing listing operations across {} service...".format(
                    len(aws_services)),
                "HEADER",
            )
        with ThreadPoolExecutor(PARALLEL_SERVICE_CALLS) as executor:
            results = executor.map(
                lambda aws_service: self.analyze_service(
                    aws_service, boto_loader, allowed_actions),
                aws_services,
            )
        for service_resources in results:
            if service_resources is not None:
                resources.extend(service_resources)

        return resources
Esempio n. 3
0
class ServiceWaiterFunctionalTest(BaseEnvVar):
    """
    This class is used as a base class if you want to functionally test the
    waiters for a specific service.
    """
    def setUp(self):
        super(ServiceWaiterFunctionalTest, self).setUp()
        self.data_path = os.path.join(os.path.dirname(botocore.__file__),
                                      'data')
        self.environ['AWS_DATA_PATH'] = self.data_path
        self.loader = Loader([self.data_path])

    def get_waiter_model(self, service, api_version=None):
        """Get the waiter model for the service."""
        with mock.patch('botocore.loaders.Loader.list_available_services',
                        return_value=[service]):
            return WaiterModel(
                self.loader.load_service_model(service,
                                               type_name='waiters-2',
                                               api_version=api_version))

    def get_service_model(self, service, api_version=None):
        """Get the service model for the service."""
        with mock.patch('botocore.loaders.Loader.list_available_services',
                        return_value=[service]):
            return ServiceModel(self.loader.load_service_model(
                service, type_name='service-2', api_version=api_version),
                                service_name=service)
Esempio n. 4
0
 def test_search_path_on_windows(self):
     # On windows, the search path is separated by ';' chars.
     self.environ['BOTO_DATA_PATH'] = 'c:\\path1;c:\\path2'
     # The builtin botocore data path is added as the last element
     # so we're only interested in checking the two that we've added.
     loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
     paths = loader.get_search_paths()[:-1]
     self.assertEqual(paths, ['c:\\path1', 'c:\\path2'])
Esempio n. 5
0
    def setUp(self):
        super(LoaderTestCase, self).setUp()
        self.data_path = os.path.join(os.path.dirname(__file__), 'data')
        self.environ['BOTO_DATA_PATH'] = self.data_path
        self.loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])

        # Make sure the cache is clear.
        self.loader._cache.clear()
Esempio n. 6
0
 def test_search_path_on_windows(self):
     # On windows, the search path is separated by ';' chars.
     self.environ['BOTO_DATA_PATH'] = 'c:\\path1;c:\\path2'
     # The builtin botocore data path is added as the last element
     # so we're only interested in checking the two that we've added.
     loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
     paths = loader.get_search_paths()[:-1]
     self.assertEqual(paths, ['c:\\path1', 'c:\\path2'])
Esempio n. 7
0
 def test_data_not_found_raises_exception(self):
     class FakeLoader(object):
         def load_file(self, name):
             # Returning None indicates that the
             # loader couldn't find anything.
             return None
     loader = Loader(file_loader=FakeLoader())
     with self.assertRaises(DataNotFoundError):
         loader.load_data('baz')
Esempio n. 8
0
 def test_data_not_found_raises_exception(self):
     class FakeLoader(object):
         def load_file(self, name):
             # Returning None indicates that the
             # loader couldn't find anything.
             return None
     loader = Loader(file_loader=FakeLoader())
     with self.assertRaises(DataNotFoundError):
         loader.load_data('baz')
Esempio n. 9
0
    def test_load_service_model(self):
        class FakeLoader(object):
            def load_file(self, name):
                return ['loaded data']

        loader = Loader(extra_search_paths=['foo'],
                        file_loader=FakeLoader(),
                        include_default_search_paths=False)
        loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
        loaded = loader.load_service_model('baz', type_name='service-2')
        self.assertEqual(loaded, ['loaded data'])
Esempio n. 10
0
    def test_load_data_uses_loader(self):
        search_paths = ['foo', 'bar', 'baz']

        class FakeLoader(object):
            def load_file(self, name):
                expected_ending = os.path.join('bar', 'baz')
                if name.endswith(expected_ending):
                    return ['loaded data']

        loader = Loader(extra_search_paths=search_paths,
                        file_loader=FakeLoader())
        loaded = loader.load_data('baz')
        self.assertEqual(loaded, ['loaded data'])
Esempio n. 11
0
    def test_load_data_uses_loader(self):
        search_paths = ['foo', 'bar', 'baz']

        class FakeLoader(object):
            def load_file(self, name):
                expected_ending = os.path.join('bar', 'baz')
                if name.endswith(expected_ending):
                    return ['loaded data']

        loader = Loader(extra_search_paths=search_paths,
                        file_loader=FakeLoader())
        loaded = loader.load_data('baz')
        self.assertEqual(loaded, ['loaded data'])
Esempio n. 12
0
    def test_load_service_model_enforces_case(self):
        class FakeLoader(object):
            def load_file(self, name):
                return ['loaded data']

        loader = Loader(extra_search_paths=['foo'],
                        file_loader=FakeLoader(),
                        include_default_search_paths=False)
        loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
        loader.list_available_services = mock.Mock(return_value=['baz'])

        with self.assertRaises(ValidationError):
            loader.load_service_model('BAZ', type_name='service-2')
Esempio n. 13
0
    def setUp(self):
        super(TestMergeExtras, self).setUp()
        self.file_loader = mock.Mock()
        self.data_loader = Loader(extra_search_paths=['datapath'],
                                  file_loader=self.file_loader,
                                  include_default_search_paths=False)
        self.data_loader.determine_latest_version = mock.Mock(
            return_value='2015-03-01')
        self.data_loader.list_available_services = mock.Mock(
            return_value=['myservice'])

        isdir_mock = mock.Mock(return_value=True)
        self.isdir_patch = mock.patch('os.path.isdir', isdir_mock)
        self.isdir_patch.start()
Esempio n. 14
0
    def test_load_service_model_uses_provided_type_name(self):
        loader = Loader(extra_search_paths=['foo'],
                        file_loader=mock.Mock(),
                        include_default_search_paths=False)
        loader.list_available_services = mock.Mock(return_value=['baz'])

        # Should have a) the unknown service name and b) list of valid
        # service names.
        provided_type_name = 'not-service-2'
        with self.assertRaisesRegex(UnknownServiceError,
                                    'Unknown service.*BAZ.*baz'):
            loader.load_service_model('BAZ', type_name=provided_type_name)

        loader.list_available_services.assert_called_with(provided_type_name)
    def test_load_service_model_uses_provided_type_name(self):
        loader = Loader(extra_search_paths=['foo'],
                        file_loader=mock.Mock(),
                        include_default_search_paths=False)
        loader.list_available_services = mock.Mock(return_value=['baz'])

        # Should have a) the unknown service name and b) list of valid
        # service names.
        provided_type_name = 'not-service-2'
        with self.assertRaisesRegexp(UnknownServiceError,
                                     'Unknown service.*BAZ.*baz'):
            loader.load_service_model(
                'BAZ', type_name=provided_type_name)

        loader.list_available_services.assert_called_with(provided_type_name)
Esempio n. 16
0
 def test_load_data_overridden(self):
     self.overrides_path = os.path.join(os.path.dirname(__file__),
                                        'data_overrides')
     self.environ['BOTO_DATA_PATH'] = "{0}{1}{2}".format(
         self.overrides_path, os.pathsep, self.data_path)
     loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
     # This should load the data the first data it finds.
     data = loader.load_service_model('someservice',
                                      api_version='2012-10-01')
     # An overridden key.
     self.assertEqual(data['api_version'], '2012-10-01')
     # A key unique to the base.
     self.assertEqual(data['something-else'], 'another')
     # Ensure a key present in other variants is not there.
     self.assertTrue('Purpose' not in data)
Esempio n. 17
0
    def setUp(self):
        self.root_dir = tempfile.mkdtemp()
        self.version_dirs = os.path.join(self.root_dir, 'myservice',
                                         '2014-01-01')
        os.makedirs(self.version_dirs)

        self.model_file = os.path.join(self.version_dirs, 'service-2.json')
        self.waiter_model_file = os.path.join(self.version_dirs,
                                              'waiters-2.json')
        self.paginator_model_file = os.path.join(self.version_dirs,
                                                 'paginators-1.json')
        self.resource_model_file = os.path.join(self.version_dirs,
                                                'resources-1.json')
        self.example_model_file = os.path.join(self.version_dirs,
                                               'examples-1.json')

        self.json_model = {}
        self.waiter_json_model = {}
        self.paginator_json_model = {}
        self.resource_json_model = {}
        self._setup_models()
        self._write_models()

        self.doc_name = 'MyDoc'
        self.doc_structure = DocumentStructure(self.doc_name)

        self.loader = Loader(extra_search_paths=[self.root_dir])
        self.botocore_session = botocore.session.get_session()
        self.botocore_session.register_component('data_loader', self.loader)
        self.session = Session(botocore_session=self.botocore_session,
                               region_name='us-east-1')
        self.client = self.session.client('myservice', 'us-east-1')
        self.resource = self.session.resource('myservice', 'us-east-1')
Esempio n. 18
0
    def setup_client(self):
        with open(self.example_model_file, 'w') as f:
            json.dump(self.example_json_model, f)

        with open(self.waiter_model_file, 'w') as f:
            json.dump(self.waiter_json_model, f)

        with open(self.paginator_model_file, 'w') as f:
            json.dump(self.paginator_json_model, f)

        with open(self.model_file, 'w') as f:
            json.dump(self.json_model, f)

        self.loader = Loader(extra_search_paths=[self.root_dir])

        endpoint_resolver = mock.Mock()
        endpoint_resolver.construct_endpoint.return_value = {
            'hostname': 'foo.us-east-1',
            'partition': 'aws',
            'endpointName': 'us-east-1',
            'signatureVersions': ['v4']
        }

        self.creator = ClientCreator(loader=self.loader,
                                     endpoint_resolver=endpoint_resolver,
                                     user_agent='user-agent',
                                     event_emitter=self.events,
                                     retry_handler_factory=mock.Mock(),
                                     retry_config_translator=mock.Mock())

        self.client = self.creator.create_client('myservice', 'us-east-1')
Esempio n. 19
0
    def setup_client(self):
        with open(self.waiter_model_file, 'w') as f:
            json.dump(self.waiter_json_model, f)

        with open(self.paginator_model_file, 'w') as f:
            json.dump(self.paginator_json_model, f)

        with open(self.model_file, 'w') as f:
            json.dump(self.json_model, f)

        self.loader = Loader(extra_search_paths=[self.root_dir])

        endpoint_resolver = mock.Mock()
        endpoint_resolver.construct_endpoint.return_value = {
            'properties': {},
            'uri': 'http://foo'
        }

        self.creator = ClientCreator(
            loader=self.loader, endpoint_resolver=endpoint_resolver,
            user_agent='user-agent', event_emitter=self.events,
            retry_handler_factory=mock.Mock(),
            retry_config_translator=mock.Mock())

        self.client = self.creator.create_client('myservice', 'us-east-1')
Esempio n. 20
0
 def test_can_initialize_with_search_paths(self):
     loader = Loader(extra_search_paths=['foo', 'bar'])
     # Note that the extra search paths are before
     # the customer/builtin data paths.
     self.assertEqual(loader.search_paths, [
         'foo', 'bar', loader.CUSTOMER_DATA_PATH, loader.BUILTIN_DATA_PATH
     ])
Esempio n. 21
0
    def test_load_service_model_enforces_case(self):
        class FakeLoader(object):
            def load_file(self, name):
                return ['loaded data']

        loader = Loader(extra_search_paths=['foo'],
                        file_loader=FakeLoader(),
                        include_default_search_paths=False)
        loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
        loader.list_available_services = mock.Mock(return_value=['baz'])

        # Should have a) the unknown service name and b) list of valid
        # service names.
        with self.assertRaisesRegexp(UnknownServiceError,
                                     'Unknown service.*BAZ.*baz'):
            loader.load_service_model('BAZ', type_name='service-2')
Esempio n. 22
0
    def setUp(self):
        super(LoaderTestCase, self).setUp()
        self.data_path = os.path.join(os.path.dirname(__file__), 'data')
        self.environ['BOTO_DATA_PATH'] = self.data_path
        self.loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])

        # Make sure the cache is clear.
        self.loader._cache.clear()
def _test_model_is_not_lost(service_name, type_name,
                            previous_version, latest_version):
    # Make sure if a paginator and/or waiter exists in previous version,
    # there will be a successor existing in latest version.
    loader = Loader()
    try:
        previous = loader.load_service_model(
            service_name, type_name, previous_version)
    except DataNotFoundError:
        pass
    else:
        try:
            latest = loader.load_service_model(
                service_name, type_name, latest_version)
        except DataNotFoundError as e:
            raise AssertionError(
                "%s must exist for %s: %s" % (type_name, service_name, e))
Esempio n. 24
0
    def test_include_default_extras(self):
        self.data_loader = Loader(extra_search_paths=['datapath'],
                                  file_loader=self.file_loader,
                                  include_default_search_paths=False,
                                  include_default_extras=False)
        self.data_loader.determine_latest_version = mock.Mock(
            return_value='2015-03-01')
        self.data_loader.list_available_services = mock.Mock(
            return_value=['myservice'])

        service_data = {'foo': 'service', 'bar': 'service'}
        service_data_copy = copy.copy(service_data)
        sdk_extras = {'merge': {'foo': 'sdk'}}
        self.file_loader.load_file.side_effect = [service_data, sdk_extras]

        loaded = self.data_loader.load_service_model('myservice', 'service-2')
        self.assertEqual(loaded, service_data_copy)
Esempio n. 25
0
 def test_can_initialize_with_search_paths(self):
     loader = Loader(extra_search_paths=['foo', 'bar'])
     self.assertIn('foo', loader.search_paths)
     self.assertIn('bar', loader.search_paths)
     # We should also always add the default search
     # paths even if the loader is initialized with
     # additional search paths.
     self.assertEqual(len(loader.search_paths), 4)
Esempio n. 26
0
def test_paginators_and_waiters_are_not_lost_in_new_version(
        service_name, type_name, previous_version, latest_version):
    # Make sure if a paginator and/or waiter exists in previous version,
    # there will be a successor existing in latest version.
    loader = Loader()
    try:
        previous = loader.load_service_model(service_name, type_name,
                                             previous_version)
    except DataNotFoundError:
        pass
    else:
        try:
            latest = loader.load_service_model(service_name, type_name,
                                               latest_version)
        except DataNotFoundError as e:
            raise AssertionError(
                f"{type_name} must exist for {service_name}: {e}")
Esempio n. 27
0
def _get_data_loader():
    # Creates a botocore data loader that loads custom data files
    # FIRST, creating a precedence for custom files.
    data_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                               BOTOCORE_DATA_FOLDER_NAME)

    return Loader(extra_search_paths=[data_folder, Loader.BUILTIN_DATA_PATH],
                  include_default_search_paths=False)
Esempio n. 28
0
 def setup_client_and_resource(self):
     self._write_models()
     self.loader = Loader(extra_search_paths=[self.root_dir])
     self.botocore_session = botocore.session.get_session()
     self.botocore_session.register_component('data_loader', self.loader)
     self.session = Session(
         botocore_session=self.botocore_session, region_name='us-east-1')
     self.client = self.session.client('myservice', 'us-east-1')
     self.resource = self.session.resource('myservice', 'us-east-1')
Esempio n. 29
0
 def loader_with_fake_dirs(self):
     mock_file_loader = mock.Mock()
     mock_file_loader.exists = self.fake_exists
     search_paths = list(self.fake_directories)
     loader = Loader(extra_search_paths=search_paths,
                     include_default_search_paths=False,
                     file_loader=mock_file_loader)
     with mock.patch('os.listdir', self.fake_listdir):
         with mock.patch('os.path.isdir', mock.Mock(return_value=True)):
             yield loader
Esempio n. 30
0
 def test_default_search_paths(self):
     loader = Loader()
     self.assertEqual(len(loader.search_paths), 2)
     # We should also have ~/.aws/models added to
     # the search path.  To deal with cross platform
     # issues we'll just check for a path that ends
     # with .aws/models.
     home_dir_path = os.path.join('.aws', 'models')
     self.assertTrue(
         any(p.endswith(home_dir_path) for p in loader.search_paths))
Esempio n. 31
0
    def setUp(self):
        self.session = get_session()
        self.files = FileCreator()

        # Create our own loader for the unit test and not rely on the
        # customer's actual ~/.aws/models nor the builtin botocore data
        # directory.
        self.customer_data_root = os.path.join(self.files.rootdir, 'customer')
        os.mkdir(self.customer_data_root)

        self.builtin_data_root = os.path.join(self.files.rootdir, 'builtin')
        os.mkdir(self.builtin_data_root)

        self.data_loader = Loader(
            [self.customer_data_root, self.builtin_data_root],
            include_default_search_paths=False)
        self.data_loader.CUSTOMER_DATA_PATH = self.customer_data_root
        self.session.register_component('data_loader', self.data_loader)
        # Since we are using a custom data loader, we need to remove
        # retries since these try to get loaded when the service model
        # is loaded.
        self.session.unregister('service-data-loaded',
                                register_retries_for_service)

        # Add some models into the builtin model directory
        # We are going to add two models. One with a matching service name
        # and endpoint and another without.
        self.matching_service = 'matching'
        self.non_matching_service = 'nonmatching'
        self.non_matching_prefix = 'nonmatching-prefix'
        self.default_api_version = '2015-10-01'

        matching_service_path = os.path.join(self.builtin_data_root,
                                             self.matching_service,
                                             self.default_api_version,
                                             'service-2.json')
        os.makedirs(os.path.dirname(matching_service_path))

        non_matching_service_path = os.path.join(self.builtin_data_root,
                                                 self.non_matching_service,
                                                 self.default_api_version,
                                                 'service-2.json')
        os.makedirs(os.path.dirname(non_matching_service_path))

        # Write the models to the builtin directory
        with open(matching_service_path, 'w') as f:
            json.dump(
                self._create_service_definition(self.matching_service,
                                                self.default_api_version), f)

        with open(non_matching_service_path, 'w') as f:
            json.dump(
                self._create_service_definition(self.non_matching_prefix,
                                                self.default_api_version), f)
Esempio n. 32
0
def _add_doctype_service_model(file_creator, session, model=None):
    if model is None:
        model = DOCTYPE_MODEL
    file_creator.create_file(
        os.path.join('doctype', '2011-06-15', 'service-2.json'),
        json.dumps(model)
    )
    data_path = session.get_config_variable('data_path').split(os.pathsep)
    loader = Loader()
    loader.search_paths.extend(data_path + [file_creator.rootdir])
    session.register_component('data_loader', loader)
Esempio n. 33
0
    def __init__(self,
                 session_vars=None,
                 event_hooks=None,
                 include_builtin_handlers=True,
                 loader=None):
        """
        Create a new Session object.

        :type session_vars: dict
        :param session_vars: A dictionary that is used to override some or all
            of the environment variables associated with this session.  The
            key/value pairs defined in this dictionary will override the
            corresponding variables defined in ``SessionVariables``.

        :type event_hooks: BaseEventHooks
        :param event_hooks: The event hooks object to use. If one is not
            provided, an event hooks object will be automatically created
            for you.

        :type include_builtin_handlers: bool
        :param include_builtin_handlers: Indicates whether or not to
            automatically register builtin handlers.
        """
        self.session_var_map = copy.copy(self.SessionVariables)
        if session_vars:
            self.session_var_map.update(session_vars)
        if event_hooks is None:
            self._events = HierarchicalEmitter()
        else:
            self._events = event_hooks
        if include_builtin_handlers:
            self._register_builtin_handlers(self._events)
        self.user_agent_name = 'Botocore'
        self.user_agent_version = __version__
        self.user_agent_extra = ''
        self._profile = None
        self._config = None
        self._credentials = None
        self._profile_map = None
        self._provider = None
        # This is a dict that stores per session specific config variable
        # overrides via set_config_variable().
        self._session_instance_vars = {}
        if loader is None:
            loader = Loader()
        self._loader = loader
        # _data_paths_added is used to track whether or not we added
        # extra paths to the loader.  We will do this lazily
        # only when we ask for the loader.
        self._data_paths_added = False
        self._components = ComponentLocator()
        self._register_components()
Esempio n. 34
0
 def test_load_data_overridden(self):
     self.overrides_path = os.path.join(
         os.path.dirname(__file__),
         'data_overrides'
     )
     self.environ['BOTO_DATA_PATH'] = "{0}{1}{2}".format(
         self.overrides_path,
         os.pathsep,
         self.data_path
     )
     loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
     # This should load the data the first data it finds.
     data = loader.load_service_model(
         'someservice',
         api_version='2012-10-01'
     )
     # An overridden key.
     self.assertEqual(data['api_version'], '2012-10-01')
     # A key unique to the base.
     self.assertEqual(data['something-else'], 'another')
     # Ensure a key present in other variants is not there.
     self.assertTrue('Purpose' not in data)
Esempio n. 35
0
    def setUp(self):
        super(TestMergeExtras, self).setUp()
        self.file_loader = mock.Mock()
        self.data_loader = Loader(
            extra_search_paths=['datapath'], file_loader=self.file_loader,
            include_default_search_paths=False)
        self.data_loader.determine_latest_version = mock.Mock(
            return_value='2015-03-01')
        self.data_loader.list_available_services = mock.Mock(
            return_value=['myservice'])

        isdir_mock = mock.Mock(return_value=True)
        self.isdir_patch = mock.patch('os.path.isdir', isdir_mock)
        self.isdir_patch.start()
Esempio n. 36
0
class ServiceWaiterFunctionalTest(BaseEnvVar):
    """
    This class is used as a base class if you want to functionally test the
    waiters for a specific service.
    """
    def setUp(self):
        super(ServiceWaiterFunctionalTest, self).setUp()
        self.data_path = os.path.join(os.path.dirname(botocore.__file__),
                                      'data')
        self.environ['BOTO_DATA_PATH'] = self.data_path
        self.loader = Loader(self.data_path)

    def get_waiter_model(self, service, api_version=None):
        """
        Get the waiter model for the service
        """
        service = os.path.join('aws', service)
        model_version = self.loader.determine_latest(service, api_version)
        # Some wierd formatting required to get the name of the model
        # correct. Right now this is returned: YYYY-MM-DD.api
        # We need: YYYY-MM-DD
        model_version = ''.join(model_version.split('.')[:-1])
        waiter_model = model_version + '.waiters'
        return WaiterModel(self.loader.load_data(waiter_model))
Esempio n. 37
0
class ServiceWaiterFunctionalTest(BaseEnvVar):
    """
    This class is used as a base class if you want to functionally test the
    waiters for a specific service.
    """
    def setUp(self):
        super(ServiceWaiterFunctionalTest, self).setUp()
        self.data_path = os.path.join(
            os.path.dirname(botocore.__file__), 'data')
        self.environ['BOTO_DATA_PATH'] = self.data_path
        self.loader = Loader(self.data_path)

    def get_waiter_model(self, service, api_version=None):
        """
        Get the waiter model for the service
        """
        service = os.path.join('aws', service)
        model_version = self.loader.determine_latest(service, api_version)
        # Some wierd formatting required to get the name of the model
        # correct. Right now this is returned: YYYY-MM-DD.api
        # We need: YYYY-MM-DD
        model_version = ''.join(model_version.split('.')[:-1])
        waiter_model = model_version + '.waiters'
        return WaiterModel(self.loader.load_data(waiter_model))
Esempio n. 38
0
    def test_include_default_extras(self):
        self.data_loader = Loader(
            extra_search_paths=['datapath'], file_loader=self.file_loader,
            include_default_search_paths=False,
            include_default_extras=False)
        self.data_loader.determine_latest_version = mock.Mock(
            return_value='2015-03-01')
        self.data_loader.list_available_services = mock.Mock(
            return_value=['myservice'])

        service_data = {'foo': 'service', 'bar': 'service'}
        service_data_copy = copy.copy(service_data)
        sdk_extras = {'merge': {'foo': 'sdk'}}
        self.file_loader.load_file.side_effect = [service_data, sdk_extras]

        loaded = self.data_loader.load_service_model('myservice', 'service-2')
        self.assertEqual(loaded, service_data_copy)
Esempio n. 39
0
class ServiceWaiterFunctionalTest(BaseEnvVar):
    """
    This class is used as a base class if you want to functionally test the
    waiters for a specific service.
    """
    def setUp(self):
        super(ServiceWaiterFunctionalTest, self).setUp()
        self.data_path = os.path.join(os.path.dirname(botocore.__file__),
                                      'data')
        self.environ['BOTO_DATA_PATH'] = self.data_path
        self.loader = Loader(self.data_path)

    def get_waiter_model(self, service, api_version=None):
        """Get the waiter model for the service."""
        return WaiterModel(
            self.loader.load_service_model(service,
                                           type_name='waiters-2',
                                           api_version=api_version))
Esempio n. 40
0
    def test_load_service_model_enforces_case(self):
        class FakeLoader(object):
            def load_file(self, name):
                return ['loaded data']

        loader = Loader(extra_search_paths=['foo'],
                        file_loader=FakeLoader(),
                        include_default_search_paths=False)
        loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
        loader.list_available_services = mock.Mock(return_value=['baz'])

        with self.assertRaises(ValidationError):
            loader.load_service_model('BAZ', type_name='service-2')
Esempio n. 41
0
    def test_load_service_model_enforces_case(self):
        class FakeLoader(object):
            def load_file(self, name):
                return ['loaded data']

        loader = Loader(extra_search_paths=['foo'],
                        file_loader=FakeLoader(),
                        include_default_search_paths=False)
        loader.determine_latest_version = mock.Mock(return_value='2015-03-01')
        loader.list_available_services = mock.Mock(return_value=['baz'])

        # Should have a) the unknown service name and b) list of valid
        # service names.
        with self.assertRaisesRegex(UnknownServiceError,
                                    'Unknown service.*BAZ.*baz'):
            loader.load_service_model('BAZ', type_name='service-2')
Esempio n. 42
0
class TestMergeExtras(BaseEnvVar):
    def setUp(self):
        super(TestMergeExtras, self).setUp()
        self.file_loader = mock.Mock()
        self.data_loader = Loader(
            extra_search_paths=['datapath'], file_loader=self.file_loader,
            include_default_search_paths=False)
        self.data_loader.determine_latest_version = mock.Mock(
            return_value='2015-03-01')
        self.data_loader.list_available_services = mock.Mock(
            return_value=['myservice'])

        isdir_mock = mock.Mock(return_value=True)
        self.isdir_patch = mock.patch('os.path.isdir', isdir_mock)
        self.isdir_patch.start()

    def tearDown(self):
        super(TestMergeExtras, self).tearDown()
        self.isdir_patch.stop()

    def test_merge_extras(self):
        service_data = {'foo': 'service', 'bar': 'service'}
        sdk_extras = {'merge': {'foo': 'sdk'}}
        self.file_loader.load_file.side_effect = [service_data, sdk_extras]

        loaded = self.data_loader.load_service_model('myservice', 'service-2')
        expected = {'foo': 'sdk', 'bar': 'service'}
        self.assertEqual(loaded, expected)

        call_args = self.file_loader.load_file.call_args_list
        call_args = [c[0][0] for c in call_args]
        base_path = os.path.join('datapath', 'myservice', '2015-03-01')
        expected_call_args = [
            os.path.join(base_path, 'service-2'),
            os.path.join(base_path, 'service-2.sdk-extras')
        ]
        self.assertEqual(call_args, expected_call_args)

    def test_extras_not_found(self):
        service_data = {'foo': 'service', 'bar': 'service'}
        service_data_copy = copy.copy(service_data)
        self.file_loader.load_file.side_effect = [service_data, None]

        loaded = self.data_loader.load_service_model('myservice', 'service-2')
        self.assertEqual(loaded, service_data_copy)

    def test_no_merge_in_extras(self):
        service_data = {'foo': 'service', 'bar': 'service'}
        service_data_copy = copy.copy(service_data)
        self.file_loader.load_file.side_effect = [service_data, {}]

        loaded = self.data_loader.load_service_model('myservice', 'service-2')
        self.assertEqual(loaded, service_data_copy)

    def test_include_default_extras(self):
        self.data_loader = Loader(
            extra_search_paths=['datapath'], file_loader=self.file_loader,
            include_default_search_paths=False,
            include_default_extras=False)
        self.data_loader.determine_latest_version = mock.Mock(
            return_value='2015-03-01')
        self.data_loader.list_available_services = mock.Mock(
            return_value=['myservice'])

        service_data = {'foo': 'service', 'bar': 'service'}
        service_data_copy = copy.copy(service_data)
        sdk_extras = {'merge': {'foo': 'sdk'}}
        self.file_loader.load_file.side_effect = [service_data, sdk_extras]

        loaded = self.data_loader.load_service_model('myservice', 'service-2')
        self.assertEqual(loaded, service_data_copy)

    def test_append_extra_type(self):
        service_data = {'foo': 'service', 'bar': 'service'}
        sdk_extras = {'merge': {'foo': 'sdk'}}
        cli_extras = {'merge': {'cli': True}}
        self.file_loader.load_file.side_effect = [
            service_data, sdk_extras, cli_extras]

        self.data_loader.extras_types.append('cli')

        loaded = self.data_loader.load_service_model('myservice', 'service-2')
        expected = {'foo': 'sdk', 'bar': 'service', 'cli': True}
        self.assertEqual(loaded, expected)

        call_args = self.file_loader.load_file.call_args_list
        call_args = [c[0][0] for c in call_args]
        base_path = os.path.join('datapath', 'myservice', '2015-03-01')
        expected_call_args = [
            os.path.join(base_path, 'service-2'),
            os.path.join(base_path, 'service-2.sdk-extras'),
            os.path.join(base_path, 'service-2.cli-extras')
        ]
        self.assertEqual(call_args, expected_call_args)

    def test_sdk_empty_extras_skipped(self):
        service_data = {'foo': 'service', 'bar': 'service'}
        cli_extras = {'merge': {'foo': 'cli'}}
        self.file_loader.load_file.side_effect = [
            service_data, None, cli_extras]

        self.data_loader.extras_types.append('cli')

        loaded = self.data_loader.load_service_model('myservice', 'service-2')
        expected = {'foo': 'cli', 'bar': 'service'}
        self.assertEqual(loaded, expected)
Esempio n. 43
0
from botocore.loaders import Loader

#Make named tuple for operations:
#'http', 'name', 'documentation', 'output', 'input'

#Make named tuple for shapes:
#'required', 'members', 'member', 'type', 'min', 'enum', 'max', 'documentation', 'pattern'

#Mane named tuple for shapes.members:
#'locationName', 'shape', 'queryName', 'idempotencyToken', 'documentation'

l = Loader()
m = l.load_service_model('ec2', 'service-2')
kset = set()


def process_operations(v1):
    '''v1 is a collection of operations entities for the service.
    Inputs and outputs for c2s are regular, except:
        {"CancelConversionTask,OrderedDict([('shape', 'CancelConversionRequest')])"}
    '''
    for k2, v2 in v1.items():
        print(f'\n{k2}')

        for k3, v3 in v2.items():
            print(f'\n\t{k3}')

            if k3 == 'http':
                #OrderedDict([('method', 'POST'), ('requestUri', '/')])
                #print(f'\n\t\thttp is {v3}')
                #if v3["method"] != 'POST' or v3["requestUri"] != '/':
Esempio n. 44
0
from botocore.exceptions import ReadTimeoutError
from botocore.loaders import Loader
from botocore.model import ServiceModel

from awswrangler import _utils, exceptions
from awswrangler._config import apply_configs
from awswrangler.s3._describe import size_objects

_logger: logging.Logger = logging.getLogger(__name__)

_S3_RETRYABLE_ERRORS: Tuple[Any, Any, Any] = (socket.timeout, ConnectionError, ReadTimeoutError)

_MIN_WRITE_BLOCK: int = 5_242_880  # 5 MB (5 * 2**20)
_MIN_PARALLEL_READ_BLOCK: int = 5_242_880  # 5 MB (5 * 2**20)

_BOTOCORE_LOADER = Loader()
_S3_JSON_MODEL = _BOTOCORE_LOADER.load_service_model(service_name="s3", type_name="service-2")
_S3_SERVICE_MODEL = ServiceModel(_S3_JSON_MODEL, service_name="s3")


def _snake_to_camel_case(s: str) -> str:
    return "".join(c.title() for c in s.split("_"))


def get_botocore_valid_kwargs(function_name: str, s3_additional_kwargs: Dict[str, Any]) -> Dict[str, Any]:
    """Filter and keep only the valid botocore key arguments."""
    s3_operation_model = _S3_SERVICE_MODEL.operation_model(_snake_to_camel_case(function_name))
    allowed_kwargs = s3_operation_model.input_shape.members.keys()  # pylint: disable=E1101
    return {k: v for k, v in s3_additional_kwargs.items() if k in allowed_kwargs}

Esempio n. 45
0
 def setUp(self):
     super(ServiceWaiterFunctionalTest, self).setUp()
     self.data_path = os.path.join(
         os.path.dirname(botocore.__file__), 'data')
     self.environ['BOTO_DATA_PATH'] = self.data_path
     self.loader = Loader(self.data_path)
Esempio n. 46
0
 def test_data_path_not_required(self):
     loader = Loader()
     self.assertEqual(loader.data_path, '')
     loader.data_path = 'foo:bar'
     self.assertEqual(loader.data_path, 'foo:bar')
Esempio n. 47
0
class LoaderTestCase(BaseEnvVar):
    def setUp(self):
        super(LoaderTestCase, self).setUp()
        self.data_path = os.path.join(os.path.dirname(__file__), 'data')
        self.environ['BOTO_DATA_PATH'] = self.data_path
        self.loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])

        # Make sure the cache is clear.
        self.loader._cache.clear()

    def test_data_path_not_required(self):
        loader = Loader()
        self.assertEqual(loader.data_path, '')
        loader.data_path = 'foo:bar'
        self.assertEqual(loader.data_path, 'foo:bar')

    def test_get_search_paths(self):
        paths = self.loader.get_search_paths()
        self.assertTrue(self.data_path in paths)

    def test_determine_latest_no_version(self):
        path = self.loader.determine_latest('someservice')
        self.assertEqual(path, os.path.join('someservice',
                                            '2013-08-21.normal'))

    def test_determine_latest_with_version(self):
        path = self.loader.determine_latest(
            'someservice',
            api_version='2012-10-01'
        )
        self.assertEqual(path, os.path.join('someservice',
                                            '2012-10-01.normal'))

    def test_determine_latest_with_version_the_wrong_way(self):
        with self.assertRaises(ApiVersionNotFoundError):
            self.loader.determine_latest('someservice/2012-10-01')

    def test_determine_latest_with_version_not_found(self):
        with self.assertRaises(ApiVersionNotFoundError):
            path = self.loader.determine_latest(
                'someservice',
                api_version='2010-02-02'
            )

    def test_load_data_plain_file(self):
        data = self.loader.load_data('foo')
        self.assertEqual(data['test_key_1'], 'test_value_1')

    def test_load_data_plain_file_nonexistant(self):
        with self.assertRaises(DataNotFoundError):
            data = self.loader.load_data('i_totally_dont_exist')

    def test_load_service_model_latest_without_version(self):
        data = self.loader.load_service_model('someservice')
        self.assertEqual(data['api_version'], '2013-08-21')

    def test_load_service_model_with_version(self):
        data = self.loader.load_service_model(
            'someservice',
            api_version='2012-10-01'
        )
        self.assertEqual(data['api_version'], '2012-10-01')

    def test_load_service_model_version_not_found(self):
        with self.assertRaises(ApiVersionNotFoundError):
            data = self.loader.load_service_model(
                'someservice',
                api_version='2010-02-02'
            )

    def test_load_service_model_data_path_order(self):
        # There's an s3/ directory both in our custom BOTO_DATA_PATH
        # directory as well as in the botocore/data/ directory.
        # Our path should win since the default built in path is always
        # last.
        data = self.loader.load_service_model('aws/s3')
        self.assertTrue(data.get('WAS_OVERRIDEN_VIA_DATA_PATH'),
                        "S3 model was loaded from botocore's default "
                        "data path instead of from the BOTO_DATA_PATH"
                        " directory.")

    def test_list_available_services(self):
        avail = self.loader.list_available_services('')
        self.assertEqual(sorted(avail), [
            'aws',
            'aws',
            'someservice',
            'sub',
        ])

        aws_avail = self.loader.list_available_services('aws')
        self.assertTrue(len(aws_avail) > 10)
        self.assertTrue('ec2' in aws_avail)

    def test_load_data_overridden(self):
        self.overrides_path = os.path.join(
            os.path.dirname(__file__),
            'data_overrides'
        )
        self.environ['BOTO_DATA_PATH'] = "{0}{1}{2}".format(
            self.overrides_path,
            os.pathsep,
            self.data_path
        )
        loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
        # This should load the data the first data it finds.
        data = loader.load_service_model(
            'someservice',
            api_version='2012-10-01'
        )
        # An overridden key.
        self.assertEqual(data['api_version'], '2012-10-01')
        # A key unique to the base.
        self.assertEqual(data['something-else'], 'another')
        # Ensure a key present in other variants is not there.
        self.assertTrue('Purpose' not in data)

    @mock.patch('os.pathsep', ';')
    def test_search_path_on_windows(self):
        # On windows, the search path is separated by ';' chars.
        self.environ['BOTO_DATA_PATH'] = 'c:\\path1;c:\\path2'
        # The builtin botocore data path is added as the last element
        # so we're only interested in checking the two that we've added.
        loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
        paths = loader.get_search_paths()[:-1]
        self.assertEqual(paths, ['c:\\path1', 'c:\\path2'])
Esempio n. 48
0
 def test_error_raised_if_service_does_not_exist(self):
     loader = Loader(extra_search_paths=[],
                     include_default_search_paths=False)
     with self.assertRaises(DataNotFoundError):
         loader.determine_latest_version('unknownservice', 'service-2')