def get_object_for_url(url: ParseResult,
existing: Optional[bool] = None) -> Object:
"""
Extracts a key (object) from a given parsed s3:// URL.
:param bool existing: If True, key is expected to exist. If False, key is expected not to
exists and it will be created. If None, the key will be created if it doesn't exist.
"""
s3_resource = cast(S3ServiceResource, session.resource('s3'))
keyName = url.path[1:]
bucketName = url.netloc
# Decide if we need to override Boto's built-in URL here.
endpoint_url: Optional[str] = None
host = os.environ.get('TOIL_S3_HOST', None)
port = os.environ.get('TOIL_S3_PORT', None)
protocol = 'https'
if os.environ.get('TOIL_S3_USE_SSL', True) == 'False':
protocol = 'http'
if host:
endpoint_url = f'{protocol}://{host}' + f':{port}' if port else ''
# TODO: OrdinaryCallingFormat equivalent in boto3?
# if botoargs:
# botoargs['calling_format'] = boto.s3.connection.OrdinaryCallingFormat()
# Get the bucket's region to avoid a redirect per request
region = get_bucket_region(bucketName, endpoint_url=endpoint_url)
s3 = cast(
S3ServiceResource,
session.resource('s3', region_name=region, endpoint_url=endpoint_url))
obj = s3.Object(bucketName, keyName)
objExists = True
try:
obj.load()
except ClientError as e:
if get_error_status(e) == 404:
objExists = False
else:
raise
if existing is True and not objExists:
raise RuntimeError(
f"Key '{keyName}' does not exist in bucket '{bucketName}'.")
elif existing is False and objExists:
raise RuntimeError(f"Key '{keyName}' exists in bucket '{bucketName}'.")
if not objExists:
obj.put() # write an empty file
return obj
def delete_iam_role(
role_name: str, region: Optional[str] = None, quiet: bool = True
) -> None:
from boto.iam.connection import IAMConnection
# TODO: the Boto3 type hints are a bit oversealous here; they want hundreds
# of overloads of the client-getting methods to exist based on the literal
# string passed in, to return exactly the right kind of client or resource.
# So we end up having to wrap all the calls in casts, which kind of defeats
# the point of a nice fluent method you can call with the name of the thing
# you want; we should have been calling iam_client() and so on all along if
# we wanted MyPy to be able to understand us. So at some point we should
# consider revising our API here to be less annoying to explain to the type
# checker.
iam_client = cast(IAMClient, session.client('iam', region_name=region))
iam_resource = cast(IAMServiceResource, session.resource('iam', region_name=region))
boto_iam_connection = IAMConnection()
role = iam_resource.Role(role_name)
# normal policies
for attached_policy in role.attached_policies.all():
printq(f'Now dissociating policy: {attached_policy.policy_name} from role {role.name}', quiet)
role.detach_policy(PolicyArn=attached_policy.arn)
# inline policies
for inline_policy in role.policies.all():
printq(f'Deleting inline policy: {inline_policy.policy_name} from role {role.name}', quiet)
# couldn't find an easy way to remove inline policies with boto3; use boto
boto_iam_connection.delete_role_policy(role.name, inline_policy.policy_name)
iam_client.delete_role(RoleName=role_name)
printq(f'Role {role_name} successfully deleted.', quiet)
def delete_s3_bucket(bucket: str,
region: Optional[str],
quiet: bool = True) -> None:
"""
Delete the given S3 bucket.
"""
printq(f'Deleting s3 bucket in region "{region}": {bucket}', quiet)
s3_client = cast(S3Client, session.client('s3', region_name=region))
s3_resource = cast(S3ServiceResource,
session.resource('s3', region_name=region))
paginator = s3_client.get_paginator('list_object_versions')
try:
for response in paginator.paginate(Bucket=bucket):
# Versions and delete markers can both go in here to be deleted.
# They both have Key and VersionId, but there's no shared base type
# defined for them in the stubs to express that. See
# <https://github.com/vemel/mypy_boto3_builder/issues/123>. So we
# have to do gymnastics to get them into the same list.
to_delete: List[Dict[str, Any]] = cast(List[Dict[str, Any]], response.get('Versions', [])) + \
cast(List[Dict[str, Any]], response.get('DeleteMarkers', []))
for entry in to_delete:
printq(
f" Deleting {entry['Key']} version {entry['VersionId']}",
quiet)
s3_client.delete_object(Bucket=bucket,
Key=entry['Key'],
VersionId=entry['VersionId'])
s3_resource.Bucket(bucket).delete()
printq(f'\n * Deleted s3 bucket successfully: {bucket}\n\n', quiet)
except s3_client.exceptions.NoSuchBucket:
printq(f'\n * S3 bucket no longer exists: {bucket}\n\n', quiet)
def delete_iam_instance_profile(
instance_profile_name: str, region: Optional[str] = None, quiet: bool = True
) -> None:
iam_resource = cast(IAMServiceResource, session.resource("iam", region_name=region))
instance_profile = iam_resource.InstanceProfile(instance_profile_name)
if instance_profile.roles is not None:
for role in instance_profile.roles:
printq(f'Now dissociating role: {role.name} from instance profile {instance_profile_name}', quiet)
instance_profile.remove_role(RoleName=role.name)
instance_profile.delete()
printq(f'Instance profile "{instance_profile_name}" successfully deleted.', quiet)