def parse_client(session: Session, service_name: ServiceName, shape_parser: ShapeParser) -> Client:
"""
Parse boto3 client to a structure.
Arguments:
session -- boto3 session.
service_name -- Target service name.
Returns:
Client structure.
"""
client = get_boto3_client(session, service_name)
public_methods = get_public_methods(client)
# remove methods that will be overriden
if "get_paginator" in public_methods:
del public_methods["get_paginator"]
if "get_waiter" in public_methods:
del public_methods["get_waiter"]
result = Client(
name=Client.get_class_name(service_name),
service_name=service_name,
boto3_client=client,
)
shape_method_map = shape_parser.get_client_method_map()
result.methods.append(result.get_exceptions_property())
for method_name, public_method in public_methods.items():
if method_name in shape_method_map:
method = shape_method_map[method_name]
else:
method = parse_method("Client", method_name, public_method, service_name)
docstring = get_short_docstring(inspect.getdoc(public_method) or "")
method.docstring = docstring
result.methods.append(method)
service_model = client.meta.service_model
client_exceptions = ClientExceptionsFactory().create_client_exceptions(service_model)
for exception_class_name in dir(client_exceptions):
if exception_class_name.startswith("_"):
continue
if not exception_class_name[0].isupper():
continue
result.exceptions_class.attributes.append(
Attribute(
exception_class_name,
TypeSubscript(
Type.Type,
[InternalImport("BotocoreClientError", stringify=False)],
),
)
)
result.attributes.append(Attribute("meta", TypeClass(ClientMeta)))
return result
def test_render(self) -> None:
assert Attribute("attr",
Type.DictStrAny).render() == "attr: Dict[str, Any]"
assert (Attribute(
"attr", Type.DictStrAny,
TypeConstant("abc")).render() == "attr: Dict[str, Any]")
assert (Attribute(
"attr", Type.DictStrAny, TypeConstant("abc"),
True).render() == "attr: Dict[str, Any] # type: ignore")
def __init__(self,
name: str,
service_name: ServiceName,
boto3_client: BaseClient,
docstring: str = ""):
super().__init__(name=name, docstring=docstring)
self.service_name = service_name
self.boto3_client = boto3_client
self.exceptions_class = ClassRecord(name="Exceptions")
self.client_error_class = ClassRecord(
name="BotocoreClientError",
attributes=[
Attribute("MSG_TEMPLATE", Type.str),
],
bases=[TypeClass(BaseException)],
methods=[
Method(
name="__init__",
arguments=[
Argument("self", None),
Argument(
"error_response",
TypeSubscript(Type.Dict, [Type.str, Type.Any])),
Argument("operation_name", Type.str),
],
return_type=Type.none,
body_lines=[
"self.response: Dict[str, Any]",
"self.operation_name: str",
],
),
],
)
def __init__(
self,
name: str,
service_name: ServiceName,
boto3_service_resource: Boto3ServiceResource,
):
self.resource_meta_class = self._get_resource_meta_class(service_name)
super().__init__(
name=name,
bases=[
ExternalImport(
source=ImportString("boto3", "resources", "base"),
name="ServiceResource",
alias="Boto3ServiceResource",
)
],
attributes=[
Attribute(
"meta",
InternalImport(
self.resource_meta_class.name,
service_name,
ServiceModuleName.service_resource,
),
)
],
)
self.service_name = service_name
self.boto3_service_resource = boto3_service_resource
self.collections: list[Collection] = []
self.sub_resources: list[Resource] = []
def parse_attributes(
service_name: ServiceName, resource_name: str, resource: Boto3ServiceResource
) -> List[Attribute]:
"""
Extract attributes from boto3 resource.
Arguments:
resource -- boto3 service resource.
Returns:
A list of Attribute structures.
"""
result: List[Attribute] = []
if not resource.meta.client:
return result
if not resource.meta.resource_model:
return result
service_model = resource.meta.client.meta.service_model
if resource.meta.resource_model.shape:
shape = service_model.shape_for(resource.meta.resource_model.shape)
attributes = resource.meta.resource_model.get_attributes(shape)
for name, attribute in attributes.items():
argument_type = get_method_type_stub(service_name, resource_name, "_attributes", name)
if argument_type is None:
argument_type = get_type_from_docstring(attribute[1].type_name)
result.append(Attribute(name, argument_type))
return result
def _get_resource_meta_class(self, service_name: ServiceName) -> ClassRecord:
return ClassRecord(
name=f"{service_name.class_name}ResourceMeta",
bases=[
ExternalImport(
source=ImportString("boto3", "resources", "base"),
name="ResourceMeta",
)
],
attributes=[Attribute("client", self._get_client_import(service_name))],
)
def parse_resource(
name: str,
resource: Boto3ServiceResource,
service_name: ServiceName,
shape_parser: ShapeParser,
) -> Resource:
"""
Parse boto3 sub Resource data.
Arguments:
resource -- Original boto3 resource.
Returns:
Resource structure.
"""
result = Resource(
name=name,
service_name=service_name,
)
shape_method_map = shape_parser.get_resource_method_map(name)
public_methods = get_resource_public_methods(resource.__class__)
for method_name, public_method in public_methods.items():
if method_name in shape_method_map:
method = shape_method_map[method_name]
else:
method = parse_method(name, method_name, public_method,
service_name)
docstring = get_short_docstring(inspect.getdoc(public_method) or "")
method.docstring = "".join((
f"{docstring}\n\n" if docstring else "",
"[Show boto3 documentation]",
f"({service_name.get_boto3_doc_link(name, method_name)})\n",
"[Show boto3-stubs documentation]",
f"({service_name.get_doc_link('service_resource', name, f'{method_name} method')})",
))
result.methods.append(method)
result.attributes.extend(parse_attributes(service_name, name, resource))
result.attributes.extend(parse_identifiers(resource))
result.attributes.extend(parse_references(resource))
collections = parse_collections(name, resource, service_name, shape_parser)
for collection in collections:
result.collections.append(collection)
result.attributes.append(
Attribute(
collection.attribute_name,
InternalImport(collection.name, service_name, stringify=False),
))
return result
def parse_resource(
name: str,
resource: Boto3ServiceResource,
service_name: ServiceName,
shape_parser: ShapeParser,
) -> Resource:
"""
Parse boto3 sub Resource data.
Arguments:
resource -- Original boto3 resource.
Returns:
Resource structure.
"""
result = Resource(
name=name,
service_name=service_name,
)
shape_method_map = shape_parser.get_resource_method_map(name)
public_methods = get_resource_public_methods(resource.__class__)
for method_name, public_method in public_methods.items():
if method_name in shape_method_map:
method = shape_method_map[method_name]
else:
method = parse_method(name, method_name, public_method,
service_name)
docstring = get_short_docstring(inspect.getdoc(public_method) or "")
method.docstring = docstring
result.methods.append(method)
attributes = parse_attributes(service_name, name, resource, shape_parser)
result.attributes.extend(attributes)
identifiers = parse_identifiers(resource)
result.attributes.extend(identifiers)
references = parse_references(resource)
result.attributes.extend(references)
collections = parse_collections(name, resource, service_name, shape_parser)
for collection in collections:
result.collections.append(collection)
result.attributes.append(
Attribute(
collection.attribute_name,
InternalImport(collection.name, service_name, stringify=False),
))
return result
def parse_identifiers(resource: Boto3ServiceResource) -> List[Attribute]:
"""
Extract identifiers from boto3 resource.
Arguments:
resource -- boto3 service resource.
Returns:
A list of Attribute structures.
"""
result: List[Attribute] = []
identifiers = resource.meta.resource_model.identifiers
for identifier in identifiers:
result.append(Attribute(identifier.name, type=Type.str))
return result
def parse_resource(
name: str,
resource: Boto3ServiceResource,
service_name: ServiceName,
shape_parser: ShapeParser,
) -> Resource:
"""
Parse boto3 sub Resource data.
Arguments:
resource -- Original boto3 resource.
Returns:
Resource structure.
"""
result = Resource(
name=name,
docstring=(
f"[{name} documentation]"
f"({service_name.doc_link}.ServiceResource.{name})"
),
)
shape_methods_map = shape_parser.get_resource_method_map(name)
public_methods = get_resource_public_methods(resource.__class__)
for method_name, public_method in public_methods.items():
if method_name in shape_methods_map:
method = shape_methods_map[method_name]
else:
method = parse_method(name, method_name, public_method, service_name)
method.docstring = (
f"[{name}.{method_name} documentation]"
f"({service_name.doc_link}.{name}.{method_name})"
)
result.methods.append(method)
result.attributes.extend(parse_attributes(service_name, name, resource))
result.attributes.extend(parse_identifiers(resource))
collections = parse_collections(name, resource, service_name, shape_parser)
for collection in collections:
result.collections.append(collection)
result.attributes.append(
Attribute(
collection.attribute_name, InternalImport(collection.name, service_name)
)
)
return result
def setup_method(self):
self.class_record = ClassRecord(
name="Name",
methods=[
Method(
name="my_method",
arguments=[
Argument("self", None),
Argument("my_str", Type.str, TypeConstant("test")),
Argument("lst", Type.ListAny),
],
return_type=Type.none,
)
],
attributes=[Attribute("attr", Type.Any, Type.none)],
bases=[Type.Any],
use_alias=True,
)
def parse_references(resource: Boto3ServiceResource) -> List[Attribute]:
"""
Extract references from boto3 resource.
Arguments:
resource -- boto3 service resource.
Returns:
A list of Attribute structures.
"""
result: List[Attribute] = []
references = resource.meta.resource_model.references
for reference in references:
if not reference.resource:
continue
result.append(
Attribute(reference.name,
type=InternalImport(reference.resource.type)))
return result
def parse_references(resource: Boto3ServiceResource) -> list[Attribute]:
"""
Extract references from boto3 resource.
Arguments:
resource -- boto3 service resource.
Returns:
A list of Attribute structures.
"""
result: list[Attribute] = []
references = resource.meta.resource_model.references
for reference in references:
if not reference.resource:
continue
type_annotation: FakeAnnotation = InternalImport(
reference.resource.type)
if reference.resource.path and "[]" in reference.resource.path:
type_annotation = TypeSubscript(Type.List, [type_annotation])
result.append(
Attribute(reference.name, type_annotation=type_annotation))
return result
def parse_attributes(
service_name: ServiceName,
resource_name: str,
resource: Boto3ServiceResource,
shape_parser: ShapeParser,
) -> list[Attribute]:
"""
Extract attributes from boto3 resource.
Arguments:
resource -- boto3 service resource.
Returns:
A list of Attribute structures.
"""
result: list[Attribute] = []
if not resource.meta.client:
return result
if not resource.meta.resource_model:
return result
if not resource.meta.resource_model.shape:
return result
service_model = resource.meta.client.meta.service_model
shape = service_model.shape_for(resource.meta.resource_model.shape)
attributes = resource.meta.resource_model.get_attributes(shape)
for name, attribute in attributes.items():
attribute_type = get_method_type_stub(service_name, resource_name,
"_attributes", name)
if attribute_type is None:
attribute_shape = attribute[1]
attribute_type = shape_parser.parse_shape(attribute_shape,
output=True)
result.append(Attribute(name, attribute_type))
return result
def test_init(self) -> None:
assert Attribute("attr", Type.DictStrAny)
assert Attribute("attr", Type.DictStrAny, TypeConstant("abc"))
assert Attribute("attr", Type.DictStrAny, TypeConstant("abc"), True)
def parse_client(session: Session, service_name: ServiceName,
shape_parser: ShapeParser) -> Client:
"""
Parse boto3 client to a structure.
Arguments:
session -- boto3 session.
service_name -- Target service name.
Returns:
Client structure.
"""
client = get_boto3_client(session, service_name)
public_methods = get_public_methods(client)
# remove methods that will be overriden
if "get_paginator" in public_methods:
del public_methods["get_paginator"]
if "get_waiter" in public_methods:
del public_methods["get_waiter"]
result = Client(
name=f"{service_name.class_name}Client",
service_name=service_name,
boto3_client=client,
docstring=(f"[{service_name.class_name}.Client documentation]"
f"({service_name.doc_link}.Client)"),
)
shape_method_map = shape_parser.get_client_method_map()
for method_name, public_method in public_methods.items():
if method_name in shape_method_map:
method = shape_method_map[method_name]
else:
method = parse_method("Client", method_name, public_method,
service_name)
method.docstring = (f"[Client.{method_name} documentation]"
f"({service_name.doc_link}.Client.{method_name})")
result.methods.append(method)
service_model = client.meta.service_model
client_exceptions = ClientExceptionsFactory().create_client_exceptions(
service_model)
for exception_class_name in dir(client_exceptions):
if exception_class_name.startswith("_"):
continue
if not exception_class_name[0].isupper():
continue
result.exceptions_class.attributes.append(
Attribute(
exception_class_name,
TypeSubscript(
Type.Type,
[TypeClass(ClientError, alias="Boto3ClientError")]),
))
result.attributes.append(
Attribute(
"exceptions",
InternalImport(
name=result.exceptions_class.name,
module_name=ServiceModuleName.client,
service_name=service_name,
stringify=False,
),
))
return result
def parse_service_resource(
session: Session, service_name: ServiceName,
shape_parser: ShapeParser) -> Optional[ServiceResource]:
"""
Parse boto3 ServiceResource data.
Arguments:
session -- boto3 session.
service_name -- Target service name.
Returns:
ServiceResource structure or None if service does not have a resource.
"""
service_resource = get_boto3_resource(session, service_name)
if service_resource is None:
return None
logger = get_logger()
logger.debug("Parsing ServiceResource")
result = ServiceResource(
name=f"{service_name.class_name}ServiceResource",
service_name=service_name,
boto3_service_resource=service_resource,
docstring=(f"[{service_name.class_name}.ServiceResource documentation]"
f"({service_name.doc_link}.ServiceResource)"),
)
public_methods = get_public_methods(service_resource)
shape_method_map = shape_parser.get_service_resource_method_map()
for method_name, public_method in public_methods.items():
if method_name in shape_method_map:
method = shape_method_map[method_name]
else:
method = parse_method("ServiceResource", method_name,
public_method, service_name)
method.docstring = (
f"[ServiceResource.{method_name} documentation]"
f"({service_name.doc_link}.ServiceResource.{method_name})")
result.methods.append(method)
logger.debug("Parsing ServiceResource attributes")
result.attributes.extend(
parse_attributes(service_name, "ServiceResource", service_resource))
result.attributes.extend(parse_identifiers(service_resource))
result.attributes.extend(parse_references(service_resource))
logger.debug("Parsing ServiceResource collections")
collections = parse_collections("ServiceResource", service_resource,
service_name, shape_parser)
for collection in collections:
result.collections.append(collection)
result.attributes.append(
Attribute(
collection.attribute_name,
InternalImport(
collection.name,
service_name,
stringify=False,
),
))
for sub_resource in get_sub_resources(session, service_name,
service_resource):
sub_resource_name = sub_resource.__class__.__name__.split(".", 1)[-1]
logger.debug(f"Parsing {sub_resource_name} sub resource")
result.sub_resources.append(
parse_resource(sub_resource_name, sub_resource, service_name,
shape_parser))
return result
