def test_get_collection_filter_method(self, ServiceModelMock: MagicMock) -> None: session_mock = MagicMock() service_name_mock = MagicMock() operation_model_mock = MagicMock() required_arg_shape_mock = MagicMock() optional_arg_shape_mock = MagicMock() operation_model_mock.input_shape.required_members = ["required_arg"] operation_model_mock.input_shape.members.items.return_value = [ ( "required_arg", required_arg_shape_mock, ), ( "optional_arg", optional_arg_shape_mock, ), ( "InputToken", optional_arg_shape_mock, ), ] ServiceModelMock().operation_names = ["my_operation"] ServiceModelMock().operation_model.return_value = operation_model_mock collection_mock = MagicMock() collection_mock.request.operation = "my_operation" shape_parser = ShapeParser(session_mock, service_name_mock) result = shape_parser.get_collection_filter_method( "MyCollection", collection_mock) self.assertEqual(result.name, "filter") self.assertEqual(len(result.decorators), 1) self.assertEqual(len(result.arguments), 3) self.assertEqual(result.arguments[0].name, "cls") self.assertEqual(result.arguments[1].name, "optional_arg") self.assertEqual(result.arguments[2].name, "InputToken")
def parse_fake_service_package( session: Session, service_name: ServiceName, package_data: type[BasePackageData]) -> ServicePackage: """ Create fake boto3 service module structure. Used by stubs and master package. Arguments: session -- boto3 session. service_name -- Target service name. package_data -- Package data. Returns: ServiceModule structure. """ shape_parser = ShapeParser(session, service_name) boto3_client = get_boto3_client(session, service_name) boto3_resource = get_boto3_resource(session, service_name) result = ServicePackage( data=package_data, service_name=service_name, client=Client( name=Client.get_class_name(service_name), service_name=service_name, boto3_client=boto3_client, ), ) if boto3_resource is not None: result.service_resource = ServiceResource( name=ServiceResource.get_class_name(service_name), service_name=service_name, boto3_service_resource=boto3_resource, ) waiter_names: list[str] = boto3_client.waiter_names for waiter_name in waiter_names: real_class_name = get_class_prefix(waiter_name) waiter_class_name = f"{real_class_name}Waiter" result.waiters.append( Waiter( waiter_class_name, waiter_name=waiter_name, service_name=service_name, )) for paginator_name in shape_parser.get_paginator_names(): operation_name = xform_name(paginator_name) result.paginators.append( Paginator( f"{paginator_name}Paginator", operation_name=operation_name, service_name=service_name, paginator_name=paginator_name, )) return result
def test_get_client_method_map(self, ServiceModelMock: MagicMock) -> None: session_mock = MagicMock() service_name_mock = MagicMock() ServiceModelMock().operation_names = ["my_operation"] session_mock._loader.load_service_model.return_value = {"resources": ["c", "a", "b"]} shape_parser = ShapeParser(session_mock, service_name_mock) result = shape_parser.get_client_method_map() assert "can_paginate" in result assert "generate_presigned_url" in result
def test_init(self) -> None: session_mock = MagicMock() service_name_mock = MagicMock() shape_parser = ShapeParser(session_mock, service_name_mock) self.assertEqual(shape_parser.service_name, service_name_mock) session_mock._loader.load_service_model.side_effect = UnknownServiceError( service_name="service_name", known_service_names="known_service_names", ) ShapeParser(session_mock, service_name_mock)
def parse_client(session: Session, service_name: ServiceName, shape_parser: ShapeParser) -> Client: """ Parse boto3 client to a structure. Arguments: session -- boto3 session. service_name -- Target service name. Returns: Client structure. """ client = get_boto3_client(session, service_name) public_methods = get_public_methods(client) # remove methods that will be overriden if "get_paginator" in public_methods: del public_methods["get_paginator"] if "get_waiter" in public_methods: del public_methods["get_waiter"] result = Client( name=Client.get_class_name(service_name), service_name=service_name, boto3_client=client, ) shape_method_map = shape_parser.get_client_method_map() result.methods.append(result.get_exceptions_property()) for method_name, public_method in public_methods.items(): if method_name in shape_method_map: method = shape_method_map[method_name] else: method = parse_method("Client", method_name, public_method, service_name) docstring = get_short_docstring(inspect.getdoc(public_method) or "") method.docstring = docstring result.methods.append(method) service_model = client.meta.service_model client_exceptions = ClientExceptionsFactory().create_client_exceptions(service_model) for exception_class_name in dir(client_exceptions): if exception_class_name.startswith("_"): continue if not exception_class_name[0].isupper(): continue result.exceptions_class.attributes.append( Attribute( exception_class_name, TypeSubscript( Type.Type, [InternalImport("BotocoreClientError", stringify=False)], ), ) ) result.attributes.append(Attribute("meta", TypeClass(ClientMeta))) return result
def test_get_paginate_method(self, ServiceModelMock: MagicMock) -> None: session_mock = MagicMock() service_name_mock = MagicMock() operation_model_mock = MagicMock() required_arg_shape_mock = MagicMock() optional_arg_shape_mock = MagicMock() operation_model_mock.input_shape.members.items.return_value = [ ( "required_arg", required_arg_shape_mock, ), ( "optional_arg", optional_arg_shape_mock, ), ( "InputToken", optional_arg_shape_mock, ), ( "skip_arg", optional_arg_shape_mock, ), ] ServiceModelMock().operation_names = ["my_paginator"] ServiceModelMock().operation_model.return_value = operation_model_mock session_mock._loader.load_service_model.return_value = { "pagination": { "my_paginator": { "input_token": "InputToken", "limit_key": "skip_arg" } }, "resources": [], } shape_parser = ShapeParser(session_mock, service_name_mock) result = shape_parser.get_paginate_method("my_paginator") self.assertEqual(result.name, "paginate") self.assertEqual(len(result.arguments), 4) self.assertEqual(result.arguments[0].name, "self") self.assertEqual(result.arguments[1].name, "required_arg") self.assertEqual(result.arguments[2].name, "optional_arg") self.assertEqual(result.arguments[3].name, "PaginationConfig")
def parse_fake_service_package(session: Session, service_name: ServiceName) -> ServicePackage: """ Create fake boto3 service module structure. Used by stubs and master package. Arguments: session -- boto3 session. service_name -- Target service name. Returns: ServiceModule structure. """ shape_parser = ShapeParser(session, service_name) result = ServicePackage( name=service_name.module_name, pypi_name=service_name.pypi_name, service_name=service_name, client=Client(), ) boto3_client = get_boto3_client(session, service_name) boto3_resource = get_boto3_resource(session, service_name) if boto3_resource is not None: result.service_resource = ServiceResource() for waiter_name in boto3_client.waiter_names: real_class_name = get_class_prefix(waiter_name) waiter_class_name = f"{real_class_name}Waiter" result.waiters.append( Waiter(waiter_class_name, waiter_name=waiter_name)) for paginator_name in shape_parser.get_paginator_names(): operation_name = xform_name(paginator_name) result.paginators.append( Paginator(f"{paginator_name}Paginator", operation_name=operation_name)) return result
def parse_resource( name: str, resource: Boto3ServiceResource, service_name: ServiceName, shape_parser: ShapeParser, ) -> Resource: """ Parse boto3 sub Resource data. Arguments: resource -- Original boto3 resource. Returns: Resource structure. """ result = Resource( name=name, service_name=service_name, ) shape_method_map = shape_parser.get_resource_method_map(name) public_methods = get_resource_public_methods(resource.__class__) for method_name, public_method in public_methods.items(): if method_name in shape_method_map: method = shape_method_map[method_name] else: method = parse_method(name, method_name, public_method, service_name) docstring = get_short_docstring(inspect.getdoc(public_method) or "") method.docstring = "".join(( f"{docstring}\n\n" if docstring else "", "[Show boto3 documentation]", f"({service_name.get_boto3_doc_link(name, method_name)})\n", "[Show boto3-stubs documentation]", f"({service_name.get_doc_link('service_resource', name, f'{method_name} method')})", )) result.methods.append(method) result.attributes.extend(parse_attributes(service_name, name, resource)) result.attributes.extend(parse_identifiers(resource)) result.attributes.extend(parse_references(resource)) collections = parse_collections(name, resource, service_name, shape_parser) for collection in collections: result.collections.append(collection) result.attributes.append( Attribute( collection.attribute_name, InternalImport(collection.name, service_name, stringify=False), )) return result
def parse_resource( name: str, resource: Boto3ServiceResource, service_name: ServiceName, shape_parser: ShapeParser, ) -> Resource: """ Parse boto3 sub Resource data. Arguments: resource -- Original boto3 resource. Returns: Resource structure. """ result = Resource( name=name, service_name=service_name, ) shape_method_map = shape_parser.get_resource_method_map(name) public_methods = get_resource_public_methods(resource.__class__) for method_name, public_method in public_methods.items(): if method_name in shape_method_map: method = shape_method_map[method_name] else: method = parse_method(name, method_name, public_method, service_name) docstring = get_short_docstring(inspect.getdoc(public_method) or "") method.docstring = docstring result.methods.append(method) attributes = parse_attributes(service_name, name, resource, shape_parser) result.attributes.extend(attributes) identifiers = parse_identifiers(resource) result.attributes.extend(identifiers) references = parse_references(resource) result.attributes.extend(references) collections = parse_collections(name, resource, service_name, shape_parser) for collection in collections: result.collections.append(collection) result.attributes.append( Attribute( collection.attribute_name, InternalImport(collection.name, service_name, stringify=False), )) return result
def parse_resource( name: str, resource: Boto3ServiceResource, service_name: ServiceName, shape_parser: ShapeParser, ) -> Resource: """ Parse boto3 sub Resource data. Arguments: resource -- Original boto3 resource. Returns: Resource structure. """ result = Resource( name=name, docstring=( f"[{name} documentation]" f"({service_name.doc_link}.ServiceResource.{name})" ), ) shape_methods_map = shape_parser.get_resource_method_map(name) public_methods = get_resource_public_methods(resource.__class__) for method_name, public_method in public_methods.items(): if method_name in shape_methods_map: method = shape_methods_map[method_name] else: method = parse_method(name, method_name, public_method, service_name) method.docstring = ( f"[{name}.{method_name} documentation]" f"({service_name.doc_link}.{name}.{method_name})" ) result.methods.append(method) result.attributes.extend(parse_attributes(service_name, name, resource)) result.attributes.extend(parse_identifiers(resource)) collections = parse_collections(name, resource, service_name, shape_parser) for collection in collections: result.collections.append(collection) result.attributes.append( Attribute( collection.attribute_name, InternalImport(collection.name, service_name) ) ) return result
def test_get_paginator_names(self) -> None: session_mock = MagicMock() service_name_mock = MagicMock() session_mock._loader.load_service_model.return_value = {"pagination": ["c", "a", "b"]} shape_parser = ShapeParser(session_mock, service_name_mock) assert shape_parser.get_paginator_names() == ["a", "b", "c"] session_mock._loader.load_service_model.return_value = {"paginations": ["c", "a", "b"]} shape_parser = ShapeParser(session_mock, service_name_mock) assert shape_parser.get_paginator_names() == []
def parse_attributes( service_name: ServiceName, resource_name: str, resource: Boto3ServiceResource, shape_parser: ShapeParser, ) -> list[Attribute]: """ Extract attributes from boto3 resource. Arguments: resource -- boto3 service resource. Returns: A list of Attribute structures. """ result: list[Attribute] = [] if not resource.meta.client: return result if not resource.meta.resource_model: return result if not resource.meta.resource_model.shape: return result service_model = resource.meta.client.meta.service_model shape = service_model.shape_for(resource.meta.resource_model.shape) attributes = resource.meta.resource_model.get_attributes(shape) for name, attribute in attributes.items(): attribute_type = get_method_type_stub(service_name, resource_name, "_attributes", name) if attribute_type is None: attribute_shape = attribute[1] attribute_type = shape_parser.parse_shape(attribute_shape, output=True) result.append(Attribute(name, attribute_type)) return result
def parse_service_package( session: Session, service_name: ServiceName, package_data: type[BasePackageData]) -> ServicePackage: """ Extract all data from boto3 service package. Arguments: session -- boto3 session. service_name -- Target service name. package_data -- Package data. Returns: ServiceModule structure. """ logger = get_logger() logger.debug("Parsing Shapes") shape_parser = ShapeParser(session, service_name) logger.debug("Parsing Client") client = parse_client(session, service_name, shape_parser) service_resource = parse_service_resource(session, service_name, shape_parser) result = ServicePackage( data=package_data, service_name=service_name, client=client, service_resource=service_resource, ) waiter_names: list[str] = client.boto3_client.waiter_names for waiter_name in waiter_names: logger.debug(f"Parsing Waiter {waiter_name}") waiter = client.boto3_client.get_waiter(waiter_name) waiter_record = Waiter( name=f"{waiter.name}Waiter", waiter_name=waiter_name, service_name=service_name, ) wait_method = shape_parser.get_wait_method(waiter.name) waiter_record.methods.append(wait_method) result.waiters.append(waiter_record) for paginator_name in shape_parser.get_paginator_names(): logger.debug(f"Parsing Paginator {paginator_name}") operation_name = xform_name(paginator_name) # boto3_paginator = client.boto3_client.get_paginator(operation_name) paginator_record = Paginator( name=f"{paginator_name}Paginator", paginator_name=paginator_name, operation_name=operation_name, service_name=service_name, ) paginate_method = shape_parser.get_paginate_method(paginator_name) paginator_record.methods.append(paginate_method) result.paginators.append(paginator_record) if result.paginators: if len(result.paginators) == 1: method = result.paginators[0].get_client_method() method.decorators.clear() result.client.methods.append(method) else: for paginator in result.paginators: method = paginator.get_client_method() result.client.methods.append(method) if result.waiters: if len(result.waiters) == 1: method = result.waiters[0].get_client_method() method.decorators.clear() result.client.methods.append(method) else: for package_waiter in result.waiters: method = package_waiter.get_client_method() result.client.methods.append(method) result.typed_dicts = result.extract_typed_dicts() result.literals = result.extract_literals() result.validate() return result
def parse_collections( parent_name: str, resource: Boto3ServiceResource, service_name: ServiceName, shape_parser: ShapeParser, ) -> List[Collection]: """ Extract collections from boto3 resource. Arguments: resource -- boto3 service resource. Returns: A list of Collection structures. """ result: List[Collection] = [] for collection in resource.meta.resource_model.collections: if not collection.resource: continue object_class_name = collection.resource.type collection_record = Collection( name=f"{parent_name}{get_class_prefix(collection.name)}Collection", parent_name=parent_name, attribute_name=collection.name, docstring=( f"[{parent_name}.{collection.name} documentation]" f"({service_name.doc_link}.{parent_name}.{collection.name})"), type=InternalImport(collection.name), ) self_type = InternalImport(collection_record.name, stringify=True) collection_record.methods.append( Method("all", [Argument("self", None)], self_type)) filter_method = shape_parser.get_collection_filter_method( collection_record.name, collection, self_type) collection_record.methods.append(filter_method) batch_methods = shape_parser.get_collection_batch_methods( collection_record.name, collection) for batch_method in batch_methods: collection_record.methods.append(batch_method) collection_record.methods.append( Method( "limit", [Argument("self", None), Argument("count", Type.int)], self_type, )) collection_record.methods.append( Method( "page_size", [Argument("self", None), Argument("count", Type.int)], self_type, )) collection_record.methods.append( Method( "pages", [Argument("self", None)], TypeSubscript( Type.Generator, [ TypeSubscript( Type.List, [InternalImport(name=object_class_name)]), Type.none, Type.none, ], ), )) collection_record.methods.append( Method( "__iter__", [Argument("self", None)], TypeSubscript( Type.Iterator, [InternalImport(name=object_class_name)], ), )) result.append(collection_record) return result
def parse_collections( parent_name: str, resource: Boto3ServiceResource, service_name: ServiceName, shape_parser: ShapeParser, ) -> List[Collection]: """ Extract collections from boto3 resource. Arguments: resource -- boto3 service resource. Returns: A list of Collection structures. """ result: List[Collection] = [] for collection in resource.meta.resource_model.collections: if not collection.resource: continue object_class_name = collection.resource.type collection_record = Collection( name=f"{parent_name}{get_class_prefix(collection.name)}Collection", parent_name=parent_name, attribute_name=collection.name, service_name=service_name, type_annotation=InternalImport(collection.name), object_class_name=object_class_name, ) self_type = InternalImport(collection_record.name, stringify=True) collection_record.methods.append( Method( name="all", arguments=[Argument("self", None)], return_type=self_type, docstring=("Get all items from the collection, optionally" " with a custom page size and item count limit."), )) filter_method = shape_parser.get_collection_filter_method( collection_record.name, collection, self_type) filter_method.docstring = ( "Get items from the collection, passing keyword arguments along" " as parameters to the underlying service operation, which are" " typically used to filter the results.") filter_method.type_ignore = True collection_record.methods.append(filter_method) batch_methods = shape_parser.get_collection_batch_methods( collection_record.name, collection) for batch_method in batch_methods: batch_method.docstring = "Batch method." collection_record.methods.append(batch_method) collection_record.methods.append( Method( "limit", [Argument("self", None), Argument("count", Type.int)], self_type, docstring=f"Return at most this many {object_class_name}s.", )) collection_record.methods.append( Method( "page_size", [Argument("self", None), Argument("count", Type.int)], self_type, docstring= f"Fetch at most this many {object_class_name}s per service request.", )) collection_record.methods.append( Method( "pages", [Argument("self", None)], TypeSubscript( Type.Iterator, [ TypeSubscript(Type.List, [InternalImport(name=object_class_name)]) ], ), docstring= f"A generator which yields pages of {object_class_name}s.", )) collection_record.methods.append( Method( "__iter__", [Argument("self", None)], TypeSubscript(Type.Iterator, [InternalImport(name=object_class_name)]), docstring=f"A generator which yields {object_class_name}s.", )) result.append(collection_record) return result
def parse_client(session: Session, service_name: ServiceName, shape_parser: ShapeParser) -> Client: """ Parse boto3 client to a structure. Arguments: session -- boto3 session. service_name -- Target service name. Returns: Client structure. """ client = get_boto3_client(session, service_name) public_methods = get_public_methods(client) # remove methods that will be overriden if "get_paginator" in public_methods: del public_methods["get_paginator"] if "get_waiter" in public_methods: del public_methods["get_waiter"] result = Client( name=f"{service_name.class_name}Client", service_name=service_name, boto3_client=client, docstring=(f"[{service_name.class_name}.Client documentation]" f"({service_name.doc_link}.Client)"), ) shape_method_map = shape_parser.get_client_method_map() for method_name, public_method in public_methods.items(): if method_name in shape_method_map: method = shape_method_map[method_name] else: method = parse_method("Client", method_name, public_method, service_name) method.docstring = (f"[Client.{method_name} documentation]" f"({service_name.doc_link}.Client.{method_name})") result.methods.append(method) service_model = client.meta.service_model client_exceptions = ClientExceptionsFactory().create_client_exceptions( service_model) for exception_class_name in dir(client_exceptions): if exception_class_name.startswith("_"): continue if not exception_class_name[0].isupper(): continue result.exceptions_class.attributes.append( Attribute( exception_class_name, TypeSubscript( Type.Type, [TypeClass(ClientError, alias="Boto3ClientError")]), )) result.attributes.append( Attribute( "exceptions", InternalImport( name=result.exceptions_class.name, module_name=ServiceModuleName.client, service_name=service_name, stringify=False, ), )) return result
def parse_service_resource( session: Session, service_name: ServiceName, shape_parser: ShapeParser) -> Optional[ServiceResource]: """ Parse boto3 ServiceResource data. Arguments: session -- boto3 session. service_name -- Target service name. Returns: ServiceResource structure or None if service does not have a resource. """ service_resource = get_boto3_resource(session, service_name) if service_resource is None: return None logger = get_logger() logger.debug("Parsing ServiceResource") result = ServiceResource( name=f"{service_name.class_name}ServiceResource", service_name=service_name, boto3_service_resource=service_resource, docstring=(f"[{service_name.class_name}.ServiceResource documentation]" f"({service_name.doc_link}.ServiceResource)"), ) public_methods = get_public_methods(service_resource) shape_method_map = shape_parser.get_service_resource_method_map() for method_name, public_method in public_methods.items(): if method_name in shape_method_map: method = shape_method_map[method_name] else: method = parse_method("ServiceResource", method_name, public_method, service_name) method.docstring = ( f"[ServiceResource.{method_name} documentation]" f"({service_name.doc_link}.ServiceResource.{method_name})") result.methods.append(method) logger.debug("Parsing ServiceResource attributes") result.attributes.extend( parse_attributes(service_name, "ServiceResource", service_resource)) result.attributes.extend(parse_identifiers(service_resource)) result.attributes.extend(parse_references(service_resource)) logger.debug("Parsing ServiceResource collections") collections = parse_collections("ServiceResource", service_resource, service_name, shape_parser) for collection in collections: result.collections.append(collection) result.attributes.append( Attribute( collection.attribute_name, InternalImport( collection.name, service_name, stringify=False, ), )) for sub_resource in get_sub_resources(session, service_name, service_resource): sub_resource_name = sub_resource.__class__.__name__.split(".", 1)[-1] logger.debug(f"Parsing {sub_resource_name} sub resource") result.sub_resources.append( parse_resource(sub_resource_name, sub_resource, service_name, shape_parser)) return result
def parse_service_package(session: Session, service_name: ServiceName) -> ServicePackage: """ Extract all data from boto3 service package. Arguments: session -- boto3 session. service_name -- Target service name. Returns: ServiceModule structure. """ logger = get_logger() logger.debug("Parsing Shapes") shape_parser = ShapeParser(session, service_name) logger.debug("Parsing Client") client = parse_client(session, service_name, shape_parser) service_resource = parse_service_resource(session, service_name, shape_parser) result = ServicePackage( name=service_name.module_name, pypi_name=service_name.pypi_name, service_name=service_name, client=client, service_resource=service_resource, ) for waiter_name in client.boto3_client.waiter_names: logger.debug(f"Parsing Waiter {waiter_name}") waiter = client.boto3_client.get_waiter(waiter_name) waiter_record = Waiter( name=f"{waiter.name}Waiter", docstring=(f"[Waiter.{waiter.name} documentation]" f"({service_name.doc_link}.Waiter.{waiter.name})"), waiter_name=waiter_name, ) wait_method = shape_parser.get_wait_method(waiter.name) wait_method.docstring = ( f"[{waiter.name}.wait documentation]" f"({service_name.doc_link}.Waiter.{waiter.name}.wait)") waiter_record.methods.append(wait_method) result.waiters.append(waiter_record) for paginator_name in shape_parser.get_paginator_names(): logger.debug(f"Parsing Paginator {paginator_name}") operation_name = xform_name(paginator_name) paginator = client.boto3_client.get_paginator(operation_name) paginator_record = Paginator( name=f"{paginator_name}Paginator", operation_name=operation_name, docstring=( f"[Paginator.{paginator_name} documentation]" f"({service_name.doc_link}.Paginator.{paginator_name})"), ) paginate_method = shape_parser.get_paginate_method(paginator_name) paginate_method.docstring = ( f"[{paginator_name}.paginate documentation]" f"({service_name.doc_link}.Paginator.{paginator_name}.paginate)") paginator_record.methods.append(paginate_method) result.paginators.append(paginator_record) for paginator in result.paginators: method = paginator.get_client_method() result.client.methods.append(method) for waiter in result.waiters: method = waiter.get_client_method() result.client.methods.append(method) result.typed_dicts = result.extract_typed_dicts(result.get_types(), {}) return result