def test_can_initialize_with_search_paths(self): loader = Loader(extra_search_paths=['foo', 'bar']) # Note that the extra search paths are before # the customer/builtin data paths. self.assertEqual(loader.search_paths, [ 'foo', 'bar', loader.CUSTOMER_DATA_PATH, loader.BUILTIN_DATA_PATH ])
def setUp(self): self.root_dir = tempfile.mkdtemp() self.version_dirs = os.path.join(self.root_dir, 'myservice', '2014-01-01') os.makedirs(self.version_dirs) self.model_file = os.path.join(self.version_dirs, 'service-2.json') self.waiter_model_file = os.path.join(self.version_dirs, 'waiters-2.json') self.paginator_model_file = os.path.join(self.version_dirs, 'paginators-1.json') self.resource_model_file = os.path.join(self.version_dirs, 'resources-1.json') self.example_model_file = os.path.join(self.version_dirs, 'examples-1.json') self.json_model = {} self.waiter_json_model = {} self.paginator_json_model = {} self.resource_json_model = {} self._setup_models() self._write_models() self.doc_name = 'MyDoc' self.doc_structure = DocumentStructure(self.doc_name) self.loader = Loader(extra_search_paths=[self.root_dir]) self.botocore_session = botocore.session.get_session() self.botocore_session.register_component('data_loader', self.loader) self.session = Session(botocore_session=self.botocore_session, region_name='us-east-1') self.client = self.session.client('myservice', 'us-east-1') self.resource = self.session.resource('myservice', 'us-east-1')
def setup_client(self): with open(self.waiter_model_file, 'w') as f: json.dump(self.waiter_json_model, f) with open(self.paginator_model_file, 'w') as f: json.dump(self.paginator_json_model, f) with open(self.model_file, 'w') as f: json.dump(self.json_model, f) self.loader = Loader(extra_search_paths=[self.root_dir]) endpoint_resolver = mock.Mock() endpoint_resolver.construct_endpoint.return_value = { 'properties': {}, 'uri': 'http://foo' } self.creator = ClientCreator( loader=self.loader, endpoint_resolver=endpoint_resolver, user_agent='user-agent', event_emitter=self.events, retry_handler_factory=mock.Mock(), retry_config_translator=mock.Mock()) self.client = self.creator.create_client('myservice', 'us-east-1')
def get_resources(self) -> List[Resource]: boto_loader = Loader() if self.options.services: aws_services = self.options.services else: aws_services = boto_loader.list_available_services( type_name="service-2") resources = [] allowed_actions = self.get_policies_allowed_actions() if self.options.verbose: message_handler( "Analyzing listing operations across {} service...".format( len(aws_services)), "HEADER", ) with ThreadPoolExecutor(PARALLEL_SERVICE_CALLS) as executor: results = executor.map( lambda aws_service: self.analyze_service( aws_service, boto_loader, allowed_actions), aws_services, ) for service_resources in results: if service_resources is not None: resources.extend(service_resources) return resources
def setup_client(self): with open(self.example_model_file, 'w') as f: json.dump(self.example_json_model, f) with open(self.waiter_model_file, 'w') as f: json.dump(self.waiter_json_model, f) with open(self.paginator_model_file, 'w') as f: json.dump(self.paginator_json_model, f) with open(self.model_file, 'w') as f: json.dump(self.json_model, f) self.loader = Loader(extra_search_paths=[self.root_dir]) endpoint_resolver = mock.Mock() endpoint_resolver.construct_endpoint.return_value = { 'hostname': 'foo.us-east-1', 'partition': 'aws', 'endpointName': 'us-east-1', 'signatureVersions': ['v4'] } self.creator = ClientCreator(loader=self.loader, endpoint_resolver=endpoint_resolver, user_agent='user-agent', event_emitter=self.events, retry_handler_factory=mock.Mock(), retry_config_translator=mock.Mock()) self.client = self.creator.create_client('myservice', 'us-east-1')
def test_can_initialize_with_search_paths(self): loader = Loader(extra_search_paths=['foo', 'bar']) self.assertIn('foo', loader.search_paths) self.assertIn('bar', loader.search_paths) # We should also always add the default search # paths even if the loader is initialized with # additional search paths. self.assertEqual(len(loader.search_paths), 4)
def setUp(self): super(LoaderTestCase, self).setUp() self.data_path = os.path.join(os.path.dirname(__file__), 'data') self.environ['BOTO_DATA_PATH'] = self.data_path self.loader = Loader(data_path=self.environ['BOTO_DATA_PATH']) # Make sure the cache is clear. self.loader._cache.clear()
def test_search_path_on_windows(self): # On windows, the search path is separated by ';' chars. self.environ['BOTO_DATA_PATH'] = 'c:\\path1;c:\\path2' # The builtin botocore data path is added as the last element # so we're only interested in checking the two that we've added. loader = Loader(data_path=self.environ['BOTO_DATA_PATH']) paths = loader.get_search_paths()[:-1] self.assertEqual(paths, ['c:\\path1', 'c:\\path2'])
def _get_data_loader(): # Creates a botocore data loader that loads custom data files # FIRST, creating a precedence for custom files. data_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), BOTOCORE_DATA_FOLDER_NAME) return Loader(extra_search_paths=[data_folder, Loader.BUILTIN_DATA_PATH], include_default_search_paths=False)
def setup_client_and_resource(self): self._write_models() self.loader = Loader(extra_search_paths=[self.root_dir]) self.botocore_session = botocore.session.get_session() self.botocore_session.register_component('data_loader', self.loader) self.session = Session( botocore_session=self.botocore_session, region_name='us-east-1') self.client = self.session.client('myservice', 'us-east-1') self.resource = self.session.resource('myservice', 'us-east-1')
def test_data_not_found_raises_exception(self): class FakeLoader(object): def load_file(self, name): # Returning None indicates that the # loader couldn't find anything. return None loader = Loader(file_loader=FakeLoader()) with self.assertRaises(DataNotFoundError): loader.load_data('baz')
def loader_with_fake_dirs(self): mock_file_loader = mock.Mock() mock_file_loader.exists = self.fake_exists search_paths = list(self.fake_directories) loader = Loader(extra_search_paths=search_paths, include_default_search_paths=False, file_loader=mock_file_loader) with mock.patch('os.listdir', self.fake_listdir): with mock.patch('os.path.isdir', mock.Mock(return_value=True)): yield loader
def test_default_search_paths(self): loader = Loader() self.assertEqual(len(loader.search_paths), 2) # We should also have ~/.aws/models added to # the search path. To deal with cross platform # issues we'll just check for a path that ends # with .aws/models. home_dir_path = os.path.join('.aws', 'models') self.assertTrue( any(p.endswith(home_dir_path) for p in loader.search_paths))
def _add_doctype_service_model(file_creator, session, model=None): if model is None: model = DOCTYPE_MODEL file_creator.create_file( os.path.join('doctype', '2011-06-15', 'service-2.json'), json.dumps(model) ) data_path = session.get_config_variable('data_path').split(os.pathsep) loader = Loader() loader.search_paths.extend(data_path + [file_creator.rootdir]) session.register_component('data_loader', loader)
def test_load_service_model(self): class FakeLoader(object): def load_file(self, name): return ['loaded data'] loader = Loader(extra_search_paths=['foo'], file_loader=FakeLoader(), include_default_search_paths=False) loader.determine_latest_version = mock.Mock(return_value='2015-03-01') loaded = loader.load_service_model('baz', type_name='service-2') self.assertEqual(loaded, ['loaded data'])
def setUp(self): self.session = get_session() self.files = FileCreator() # Create our own loader for the unit test and not rely on the # customer's actual ~/.aws/models nor the builtin botocore data # directory. self.customer_data_root = os.path.join(self.files.rootdir, 'customer') os.mkdir(self.customer_data_root) self.builtin_data_root = os.path.join(self.files.rootdir, 'builtin') os.mkdir(self.builtin_data_root) self.data_loader = Loader( [self.customer_data_root, self.builtin_data_root], include_default_search_paths=False) self.data_loader.CUSTOMER_DATA_PATH = self.customer_data_root self.session.register_component('data_loader', self.data_loader) # Since we are using a custom data loader, we need to remove # retries since these try to get loaded when the service model # is loaded. self.session.unregister('service-data-loaded', register_retries_for_service) # Add some models into the builtin model directory # We are going to add two models. One with a matching service name # and endpoint and another without. self.matching_service = 'matching' self.non_matching_service = 'nonmatching' self.non_matching_prefix = 'nonmatching-prefix' self.default_api_version = '2015-10-01' matching_service_path = os.path.join(self.builtin_data_root, self.matching_service, self.default_api_version, 'service-2.json') os.makedirs(os.path.dirname(matching_service_path)) non_matching_service_path = os.path.join(self.builtin_data_root, self.non_matching_service, self.default_api_version, 'service-2.json') os.makedirs(os.path.dirname(non_matching_service_path)) # Write the models to the builtin directory with open(matching_service_path, 'w') as f: json.dump( self._create_service_definition(self.matching_service, self.default_api_version), f) with open(non_matching_service_path, 'w') as f: json.dump( self._create_service_definition(self.non_matching_prefix, self.default_api_version), f)
def __init__(self, session_vars=None, event_hooks=None, include_builtin_handlers=True, loader=None): """ Create a new Session object. :type session_vars: dict :param session_vars: A dictionary that is used to override some or all of the environment variables associated with this session. The key/value pairs defined in this dictionary will override the corresponding variables defined in ``SessionVariables``. :type event_hooks: BaseEventHooks :param event_hooks: The event hooks object to use. If one is not provided, an event hooks object will be automatically created for you. :type include_builtin_handlers: bool :param include_builtin_handlers: Indicates whether or not to automatically register builtin handlers. """ self.session_var_map = copy.copy(self.SessionVariables) if session_vars: self.session_var_map.update(session_vars) if event_hooks is None: self._events = HierarchicalEmitter() else: self._events = event_hooks if include_builtin_handlers: self._register_builtin_handlers(self._events) self.user_agent_name = 'Botocore' self.user_agent_version = __version__ self.user_agent_extra = '' self._profile = None self._config = None self._credentials = None self._profile_map = None self._provider = None # This is a dict that stores per session specific config variable # overrides via set_config_variable(). self._session_instance_vars = {} if loader is None: loader = Loader() self._loader = loader # _data_paths_added is used to track whether or not we added # extra paths to the loader. We will do this lazily # only when we ask for the loader. self._data_paths_added = False self._components = ComponentLocator() self._register_components()
def test_load_data_uses_loader(self): search_paths = ['foo', 'bar', 'baz'] class FakeLoader(object): def load_file(self, name): expected_ending = os.path.join('bar', 'baz') if name.endswith(expected_ending): return ['loaded data'] loader = Loader(extra_search_paths=search_paths, file_loader=FakeLoader()) loaded = loader.load_data('baz') self.assertEqual(loaded, ['loaded data'])
def test_load_service_model_enforces_case(self): class FakeLoader(object): def load_file(self, name): return ['loaded data'] loader = Loader(extra_search_paths=['foo'], file_loader=FakeLoader(), include_default_search_paths=False) loader.determine_latest_version = mock.Mock(return_value='2015-03-01') loader.list_available_services = mock.Mock(return_value=['baz']) with self.assertRaises(ValidationError): loader.load_service_model('BAZ', type_name='service-2')
def setUp(self): super(TestMergeExtras, self).setUp() self.file_loader = mock.Mock() self.data_loader = Loader(extra_search_paths=['datapath'], file_loader=self.file_loader, include_default_search_paths=False) self.data_loader.determine_latest_version = mock.Mock( return_value='2015-03-01') self.data_loader.list_available_services = mock.Mock( return_value=['myservice']) isdir_mock = mock.Mock(return_value=True) self.isdir_patch = mock.patch('os.path.isdir', isdir_mock) self.isdir_patch.start()
def test_load_service_model_uses_provided_type_name(self): loader = Loader(extra_search_paths=['foo'], file_loader=mock.Mock(), include_default_search_paths=False) loader.list_available_services = mock.Mock(return_value=['baz']) # Should have a) the unknown service name and b) list of valid # service names. provided_type_name = 'not-service-2' with self.assertRaisesRegex(UnknownServiceError, 'Unknown service.*BAZ.*baz'): loader.load_service_model('BAZ', type_name=provided_type_name) loader.list_available_services.assert_called_with(provided_type_name)
def test_load_data_overridden(self): self.overrides_path = os.path.join(os.path.dirname(__file__), 'data_overrides') self.environ['BOTO_DATA_PATH'] = "{0}{1}{2}".format( self.overrides_path, os.pathsep, self.data_path) loader = Loader(data_path=self.environ['BOTO_DATA_PATH']) # This should load the data the first data it finds. data = loader.load_service_model('someservice', api_version='2012-10-01') # An overridden key. self.assertEqual(data['api_version'], '2012-10-01') # A key unique to the base. self.assertEqual(data['something-else'], 'another') # Ensure a key present in other variants is not there. self.assertTrue('Purpose' not in data)
def test_paginators_and_waiters_are_not_lost_in_new_version( service_name, type_name, previous_version, latest_version): # Make sure if a paginator and/or waiter exists in previous version, # there will be a successor existing in latest version. loader = Loader() try: loader.load_service_model(service_name, type_name, previous_version) except DataNotFoundError: pass else: try: loader.load_service_model(service_name, type_name, latest_version) except DataNotFoundError as e: raise AssertionError( f"{type_name} must exist for {service_name}: {e}")
def test_load_service_model_enforces_case(self): class FakeLoader(object): def load_file(self, name): return ['loaded data'] loader = Loader(extra_search_paths=['foo'], file_loader=FakeLoader(), include_default_search_paths=False) loader.determine_latest_version = mock.Mock(return_value='2015-03-01') loader.list_available_services = mock.Mock(return_value=['baz']) # Should have a) the unknown service name and b) list of valid # service names. with self.assertRaisesRegex(UnknownServiceError, 'Unknown service.*BAZ.*baz'): loader.load_service_model('BAZ', type_name='service-2')
def test_include_default_extras(self): self.data_loader = Loader(extra_search_paths=['datapath'], file_loader=self.file_loader, include_default_search_paths=False, include_default_extras=False) self.data_loader.determine_latest_version = mock.Mock( return_value='2015-03-01') self.data_loader.list_available_services = mock.Mock( return_value=['myservice']) service_data = {'foo': 'service', 'bar': 'service'} service_data_copy = copy.copy(service_data) sdk_extras = {'merge': {'foo': 'sdk'}} self.file_loader.load_file.side_effect = [service_data, sdk_extras] loaded = self.data_loader.load_service_model('myservice', 'service-2') self.assertEqual(loaded, service_data_copy)
def _register_data_loader(self): self._components.lazy_register_component( 'data_loader', lambda: Loader(self.get_config_variable('data_path') or ''))
from botocore.loaders import Loader #Make named tuple for operations: #'http', 'name', 'documentation', 'output', 'input' #Make named tuple for shapes: #'required', 'members', 'member', 'type', 'min', 'enum', 'max', 'documentation', 'pattern' #Mane named tuple for shapes.members: #'locationName', 'shape', 'queryName', 'idempotencyToken', 'documentation' l = Loader() m = l.load_service_model('ec2', 'service-2') kset = set() def process_operations(v1): '''v1 is a collection of operations entities for the service. Inputs and outputs for c2s are regular, except: {"CancelConversionTask,OrderedDict([('shape', 'CancelConversionRequest')])"} ''' for k2, v2 in v1.items(): print(f'\n{k2}') for k3, v3 in v2.items(): print(f'\n\t{k3}') if k3 == 'http': #OrderedDict([('method', 'POST'), ('requestUri', '/')]) #print(f'\n\t\thttp is {v3}') #if v3["method"] != 'POST' or v3["requestUri"] != '/':
from botocore.exceptions import ReadTimeoutError from botocore.loaders import Loader from botocore.model import ServiceModel from awswrangler import _utils, exceptions from awswrangler._config import apply_configs from awswrangler.s3._describe import size_objects _logger: logging.Logger = logging.getLogger(__name__) _S3_RETRYABLE_ERRORS: Tuple[Any, Any, Any] = (socket.timeout, ConnectionError, ReadTimeoutError) _MIN_WRITE_BLOCK: int = 5_242_880 # 5 MB (5 * 2**20) _MIN_PARALLEL_READ_BLOCK: int = 5_242_880 # 5 MB (5 * 2**20) _BOTOCORE_LOADER = Loader() _S3_JSON_MODEL = _BOTOCORE_LOADER.load_service_model(service_name="s3", type_name="service-2") _S3_SERVICE_MODEL = ServiceModel(_S3_JSON_MODEL, service_name="s3") def _snake_to_camel_case(s: str) -> str: return "".join(c.title() for c in s.split("_")) def get_botocore_valid_kwargs(function_name: str, s3_additional_kwargs: Dict[str, Any]) -> Dict[str, Any]: """Filter and keep only the valid botocore key arguments.""" s3_operation_model = _S3_SERVICE_MODEL.operation_model(_snake_to_camel_case(function_name)) allowed_kwargs = s3_operation_model.input_shape.members.keys() # pylint: disable=E1101 return {k: v for k, v in s3_additional_kwargs.items() if k in allowed_kwargs}
def test_can_add_to_search_path(self): loader = Loader() loader.search_paths.append('mypath') self.assertIn('mypath', loader.search_paths)
def test_error_raised_if_service_does_not_exist(self): loader = Loader(extra_search_paths=[], include_default_search_paths=False) with self.assertRaises(DataNotFoundError): loader.determine_latest_version('unknownservice', 'service-2')