def parse_client(session: Session, service_name: ServiceName, shape_parser: ShapeParser) -> Client: """ Parse boto3 client to a structure. Arguments: session -- boto3 session. service_name -- Target service name. Returns: Client structure. """ client = get_boto3_client(session, service_name) public_methods = get_public_methods(client) # remove methods that will be overriden if "get_paginator" in public_methods: del public_methods["get_paginator"] if "get_waiter" in public_methods: del public_methods["get_waiter"] result = Client( name=Client.get_class_name(service_name), service_name=service_name, boto3_client=client, ) shape_method_map = shape_parser.get_client_method_map() result.methods.append(result.get_exceptions_property()) for method_name, public_method in public_methods.items(): if method_name in shape_method_map: method = shape_method_map[method_name] else: method = parse_method("Client", method_name, public_method, service_name) docstring = get_short_docstring(inspect.getdoc(public_method) or "") method.docstring = docstring result.methods.append(method) service_model = client.meta.service_model client_exceptions = ClientExceptionsFactory().create_client_exceptions(service_model) for exception_class_name in dir(client_exceptions): if exception_class_name.startswith("_"): continue if not exception_class_name[0].isupper(): continue result.exceptions_class.attributes.append( Attribute( exception_class_name, TypeSubscript( Type.Type, [InternalImport("BotocoreClientError", stringify=False)], ), ) ) result.attributes.append(Attribute("meta", TypeClass(ClientMeta))) return result
def test_render(self) -> None: assert Attribute("attr", Type.DictStrAny).render() == "attr: Dict[str, Any]" assert (Attribute( "attr", Type.DictStrAny, TypeConstant("abc")).render() == "attr: Dict[str, Any]") assert (Attribute( "attr", Type.DictStrAny, TypeConstant("abc"), True).render() == "attr: Dict[str, Any] # type: ignore")
def __init__(self, name: str, service_name: ServiceName, boto3_client: BaseClient, docstring: str = ""): super().__init__(name=name, docstring=docstring) self.service_name = service_name self.boto3_client = boto3_client self.exceptions_class = ClassRecord(name="Exceptions") self.client_error_class = ClassRecord( name="BotocoreClientError", attributes=[ Attribute("MSG_TEMPLATE", Type.str), ], bases=[TypeClass(BaseException)], methods=[ Method( name="__init__", arguments=[ Argument("self", None), Argument( "error_response", TypeSubscript(Type.Dict, [Type.str, Type.Any])), Argument("operation_name", Type.str), ], return_type=Type.none, body_lines=[ "self.response: Dict[str, Any]", "self.operation_name: str", ], ), ], )
def __init__( self, name: str, service_name: ServiceName, boto3_service_resource: Boto3ServiceResource, ): self.resource_meta_class = self._get_resource_meta_class(service_name) super().__init__( name=name, bases=[ ExternalImport( source=ImportString("boto3", "resources", "base"), name="ServiceResource", alias="Boto3ServiceResource", ) ], attributes=[ Attribute( "meta", InternalImport( self.resource_meta_class.name, service_name, ServiceModuleName.service_resource, ), ) ], ) self.service_name = service_name self.boto3_service_resource = boto3_service_resource self.collections: list[Collection] = [] self.sub_resources: list[Resource] = []
def parse_attributes( service_name: ServiceName, resource_name: str, resource: Boto3ServiceResource ) -> List[Attribute]: """ Extract attributes from boto3 resource. Arguments: resource -- boto3 service resource. Returns: A list of Attribute structures. """ result: List[Attribute] = [] if not resource.meta.client: return result if not resource.meta.resource_model: return result service_model = resource.meta.client.meta.service_model if resource.meta.resource_model.shape: shape = service_model.shape_for(resource.meta.resource_model.shape) attributes = resource.meta.resource_model.get_attributes(shape) for name, attribute in attributes.items(): argument_type = get_method_type_stub(service_name, resource_name, "_attributes", name) if argument_type is None: argument_type = get_type_from_docstring(attribute[1].type_name) result.append(Attribute(name, argument_type)) return result
def _get_resource_meta_class(self, service_name: ServiceName) -> ClassRecord: return ClassRecord( name=f"{service_name.class_name}ResourceMeta", bases=[ ExternalImport( source=ImportString("boto3", "resources", "base"), name="ResourceMeta", ) ], attributes=[Attribute("client", self._get_client_import(service_name))], )
def parse_resource( name: str, resource: Boto3ServiceResource, service_name: ServiceName, shape_parser: ShapeParser, ) -> Resource: """ Parse boto3 sub Resource data. Arguments: resource -- Original boto3 resource. Returns: Resource structure. """ result = Resource( name=name, service_name=service_name, ) shape_method_map = shape_parser.get_resource_method_map(name) public_methods = get_resource_public_methods(resource.__class__) for method_name, public_method in public_methods.items(): if method_name in shape_method_map: method = shape_method_map[method_name] else: method = parse_method(name, method_name, public_method, service_name) docstring = get_short_docstring(inspect.getdoc(public_method) or "") method.docstring = "".join(( f"{docstring}\n\n" if docstring else "", "[Show boto3 documentation]", f"({service_name.get_boto3_doc_link(name, method_name)})\n", "[Show boto3-stubs documentation]", f"({service_name.get_doc_link('service_resource', name, f'{method_name} method')})", )) result.methods.append(method) result.attributes.extend(parse_attributes(service_name, name, resource)) result.attributes.extend(parse_identifiers(resource)) result.attributes.extend(parse_references(resource)) collections = parse_collections(name, resource, service_name, shape_parser) for collection in collections: result.collections.append(collection) result.attributes.append( Attribute( collection.attribute_name, InternalImport(collection.name, service_name, stringify=False), )) return result
def parse_resource( name: str, resource: Boto3ServiceResource, service_name: ServiceName, shape_parser: ShapeParser, ) -> Resource: """ Parse boto3 sub Resource data. Arguments: resource -- Original boto3 resource. Returns: Resource structure. """ result = Resource( name=name, service_name=service_name, ) shape_method_map = shape_parser.get_resource_method_map(name) public_methods = get_resource_public_methods(resource.__class__) for method_name, public_method in public_methods.items(): if method_name in shape_method_map: method = shape_method_map[method_name] else: method = parse_method(name, method_name, public_method, service_name) docstring = get_short_docstring(inspect.getdoc(public_method) or "") method.docstring = docstring result.methods.append(method) attributes = parse_attributes(service_name, name, resource, shape_parser) result.attributes.extend(attributes) identifiers = parse_identifiers(resource) result.attributes.extend(identifiers) references = parse_references(resource) result.attributes.extend(references) collections = parse_collections(name, resource, service_name, shape_parser) for collection in collections: result.collections.append(collection) result.attributes.append( Attribute( collection.attribute_name, InternalImport(collection.name, service_name, stringify=False), )) return result
def parse_identifiers(resource: Boto3ServiceResource) -> List[Attribute]: """ Extract identifiers from boto3 resource. Arguments: resource -- boto3 service resource. Returns: A list of Attribute structures. """ result: List[Attribute] = [] identifiers = resource.meta.resource_model.identifiers for identifier in identifiers: result.append(Attribute(identifier.name, type=Type.str)) return result
def parse_resource( name: str, resource: Boto3ServiceResource, service_name: ServiceName, shape_parser: ShapeParser, ) -> Resource: """ Parse boto3 sub Resource data. Arguments: resource -- Original boto3 resource. Returns: Resource structure. """ result = Resource( name=name, docstring=( f"[{name} documentation]" f"({service_name.doc_link}.ServiceResource.{name})" ), ) shape_methods_map = shape_parser.get_resource_method_map(name) public_methods = get_resource_public_methods(resource.__class__) for method_name, public_method in public_methods.items(): if method_name in shape_methods_map: method = shape_methods_map[method_name] else: method = parse_method(name, method_name, public_method, service_name) method.docstring = ( f"[{name}.{method_name} documentation]" f"({service_name.doc_link}.{name}.{method_name})" ) result.methods.append(method) result.attributes.extend(parse_attributes(service_name, name, resource)) result.attributes.extend(parse_identifiers(resource)) collections = parse_collections(name, resource, service_name, shape_parser) for collection in collections: result.collections.append(collection) result.attributes.append( Attribute( collection.attribute_name, InternalImport(collection.name, service_name) ) ) return result
def setup_method(self): self.class_record = ClassRecord( name="Name", methods=[ Method( name="my_method", arguments=[ Argument("self", None), Argument("my_str", Type.str, TypeConstant("test")), Argument("lst", Type.ListAny), ], return_type=Type.none, ) ], attributes=[Attribute("attr", Type.Any, Type.none)], bases=[Type.Any], use_alias=True, )
def parse_references(resource: Boto3ServiceResource) -> List[Attribute]: """ Extract references from boto3 resource. Arguments: resource -- boto3 service resource. Returns: A list of Attribute structures. """ result: List[Attribute] = [] references = resource.meta.resource_model.references for reference in references: if not reference.resource: continue result.append( Attribute(reference.name, type=InternalImport(reference.resource.type))) return result
def parse_references(resource: Boto3ServiceResource) -> list[Attribute]: """ Extract references from boto3 resource. Arguments: resource -- boto3 service resource. Returns: A list of Attribute structures. """ result: list[Attribute] = [] references = resource.meta.resource_model.references for reference in references: if not reference.resource: continue type_annotation: FakeAnnotation = InternalImport( reference.resource.type) if reference.resource.path and "[]" in reference.resource.path: type_annotation = TypeSubscript(Type.List, [type_annotation]) result.append( Attribute(reference.name, type_annotation=type_annotation)) return result
def parse_attributes( service_name: ServiceName, resource_name: str, resource: Boto3ServiceResource, shape_parser: ShapeParser, ) -> list[Attribute]: """ Extract attributes from boto3 resource. Arguments: resource -- boto3 service resource. Returns: A list of Attribute structures. """ result: list[Attribute] = [] if not resource.meta.client: return result if not resource.meta.resource_model: return result if not resource.meta.resource_model.shape: return result service_model = resource.meta.client.meta.service_model shape = service_model.shape_for(resource.meta.resource_model.shape) attributes = resource.meta.resource_model.get_attributes(shape) for name, attribute in attributes.items(): attribute_type = get_method_type_stub(service_name, resource_name, "_attributes", name) if attribute_type is None: attribute_shape = attribute[1] attribute_type = shape_parser.parse_shape(attribute_shape, output=True) result.append(Attribute(name, attribute_type)) return result
def test_init(self) -> None: assert Attribute("attr", Type.DictStrAny) assert Attribute("attr", Type.DictStrAny, TypeConstant("abc")) assert Attribute("attr", Type.DictStrAny, TypeConstant("abc"), True)
def parse_client(session: Session, service_name: ServiceName, shape_parser: ShapeParser) -> Client: """ Parse boto3 client to a structure. Arguments: session -- boto3 session. service_name -- Target service name. Returns: Client structure. """ client = get_boto3_client(session, service_name) public_methods = get_public_methods(client) # remove methods that will be overriden if "get_paginator" in public_methods: del public_methods["get_paginator"] if "get_waiter" in public_methods: del public_methods["get_waiter"] result = Client( name=f"{service_name.class_name}Client", service_name=service_name, boto3_client=client, docstring=(f"[{service_name.class_name}.Client documentation]" f"({service_name.doc_link}.Client)"), ) shape_method_map = shape_parser.get_client_method_map() for method_name, public_method in public_methods.items(): if method_name in shape_method_map: method = shape_method_map[method_name] else: method = parse_method("Client", method_name, public_method, service_name) method.docstring = (f"[Client.{method_name} documentation]" f"({service_name.doc_link}.Client.{method_name})") result.methods.append(method) service_model = client.meta.service_model client_exceptions = ClientExceptionsFactory().create_client_exceptions( service_model) for exception_class_name in dir(client_exceptions): if exception_class_name.startswith("_"): continue if not exception_class_name[0].isupper(): continue result.exceptions_class.attributes.append( Attribute( exception_class_name, TypeSubscript( Type.Type, [TypeClass(ClientError, alias="Boto3ClientError")]), )) result.attributes.append( Attribute( "exceptions", InternalImport( name=result.exceptions_class.name, module_name=ServiceModuleName.client, service_name=service_name, stringify=False, ), )) return result
def parse_service_resource( session: Session, service_name: ServiceName, shape_parser: ShapeParser) -> Optional[ServiceResource]: """ Parse boto3 ServiceResource data. Arguments: session -- boto3 session. service_name -- Target service name. Returns: ServiceResource structure or None if service does not have a resource. """ service_resource = get_boto3_resource(session, service_name) if service_resource is None: return None logger = get_logger() logger.debug("Parsing ServiceResource") result = ServiceResource( name=f"{service_name.class_name}ServiceResource", service_name=service_name, boto3_service_resource=service_resource, docstring=(f"[{service_name.class_name}.ServiceResource documentation]" f"({service_name.doc_link}.ServiceResource)"), ) public_methods = get_public_methods(service_resource) shape_method_map = shape_parser.get_service_resource_method_map() for method_name, public_method in public_methods.items(): if method_name in shape_method_map: method = shape_method_map[method_name] else: method = parse_method("ServiceResource", method_name, public_method, service_name) method.docstring = ( f"[ServiceResource.{method_name} documentation]" f"({service_name.doc_link}.ServiceResource.{method_name})") result.methods.append(method) logger.debug("Parsing ServiceResource attributes") result.attributes.extend( parse_attributes(service_name, "ServiceResource", service_resource)) result.attributes.extend(parse_identifiers(service_resource)) result.attributes.extend(parse_references(service_resource)) logger.debug("Parsing ServiceResource collections") collections = parse_collections("ServiceResource", service_resource, service_name, shape_parser) for collection in collections: result.collections.append(collection) result.attributes.append( Attribute( collection.attribute_name, InternalImport( collection.name, service_name, stringify=False, ), )) for sub_resource in get_sub_resources(session, service_name, service_resource): sub_resource_name = sub_resource.__class__.__name__.split(".", 1)[-1] logger.debug(f"Parsing {sub_resource_name} sub resource") result.sub_resources.append( parse_resource(sub_resource_name, sub_resource, service_name, shape_parser)) return result