def setUp(self):
self.service_object = mock.Mock()
# Create some waiters.
self.model = WaiterModel({
'version': 2,
'waiters': {
'InstanceRunning': {
'description': 'my waiter description',
'delay': 1,
'maxAttempts': 10,
'operation': 'MyOperation',
},
'BucketExists': {
'description': 'my waiter description',
'operation': 'MyOperation',
'delay': 1,
'maxAttempts': 10,
}
}
})
self.waiter_builder = WaiterStateCommandBuilder(
self.model,
self.service_object
)
def create_fargate_profile(
profile_name: str,
cluster_name: str,
role_arn: str,
subnets: List[str],
namespace: str,
selector_labels: Optional[Dict[str, Any]] = None,
) -> None:
_logger.debug(f"Creating EKS Fargate Profile: {profile_name}")
if describe_fargate_profile(profile_name=profile_name,
cluster_name=cluster_name) is not None:
_logger.debug(f"EKS Fargate Profile already exists: {profile_name}")
return
eks_client = boto3_client("eks")
eks_client.create_fargate_profile(
fargateProfileName=profile_name,
clusterName=cluster_name,
podExecutionRoleArn=role_arn,
subnets=subnets,
selectors=[{
"namespace": namespace,
"labels": selector_labels
}],
)
waiter_model = WaiterModel(WAITER_CONFIG)
waiter = create_waiter_with_client("FargateProfileCreated", waiter_model,
eks_client)
waiter.wait(fargateProfileName=profile_name, clusterName=cluster_name)
_logger.debug(f"Created EKS Fargate Profile: {profile_name}")
def test_unsupported_waiter_version(self):
waiters = {
'version': 1,
'waiters': {}
}
with self.assertRaises(WaiterConfigError):
WaiterModel(waiters)
def create_waiter(evb_client):
waiter_name = "EvbRuleEnabled"
waiter_config = {
"version": 2,
"waiters": {
"EvbRuleEnabled": {
"operation":
"DescribeRule",
"delay":
WAITER_DELAY,
"maxAttempts":
WAITER_MAX_ATTEMPTS,
"acceptors": [
{
"matcher": "path",
"expected": "ENABLED",
"state": "success",
"argument": "State",
},
{
"matcher": "path",
"expected": "DISABLED",
"state": "retry",
"argument": "State",
},
],
}
},
}
waiter_model = WaiterModel(waiter_config)
return create_waiter_with_client(waiter_name, waiter_model, evb_client)
def fargate_delete_waiter(self, client):
# Fargate profiles seem to delete faster @ roughly 2 minutes each so keeping defaults
config = {
'version': 2,
'waiters': {
"FargateProfileDeleted": {
'operation':
'DescribeFargateProfile',
'delay':
30,
'maxAttempts':
40,
'acceptors': [{
"expected": "DELETE_FAILED",
"matcher": "path",
"state": "failure",
"argument": "fargateprofile.status"
}, {
"expected": "ResourceNotFoundException",
"matcher": "error",
"state": "success"
}]
}
}
}
return create_waiter_with_client("FargateProfileDeleted",
WaiterModel(config), client)
def test_get_single_waiter_config(self):
single_waiter = {
'description':
'Waiter description',
'operation':
'HeadBucket',
'delay':
5,
'maxAttempts':
20,
'acceptors': [
{
'state': 'success',
'matcher': 'status',
'expected': 200
},
{
'state': 'retry',
'matcher': 'status',
'expected': 404
},
],
}
waiters = {
'version': 2,
'waiters': {
'BucketExists': single_waiter,
}
}
model = WaiterModel(waiters)
config = model.get_waiter('BucketExists')
self.assertEqual(config.operation, 'HeadBucket')
def wait_for_changes_in_aws(autoscaling_client, group_name, desired_capacity) -> None:
waiter_name = 'autoscaling_completed'
argument = f"contains(AutoScalingGroups[?(starts_with(AutoScalingGroupName, `{group_name}`) == `true`)]." \
f"[length(Instances[?LifecycleState=='InService']) == `{desired_capacity}`][], `false`)"
waiter_config = {
"version": 2,
"waiters": {
"autoscaling_completed": {
"acceptors": [
{
"argument": argument,
"expected": True,
"matcher": "path",
"state": "success"
},
{
"argument": argument,
"expected": False,
"matcher": "path",
"state": "retry"
}
],
"delay": 5,
"maxAttempts": 20,
"operation": "DescribeAutoScalingGroups"
}
}
}
waiter_model = WaiterModel(waiter_config)
custom_waiter = create_waiter_with_client(waiter_name, waiter_model, autoscaling_client)
custom_waiter.wait()
def get_compute_job_queue_waiter(self, waiter_id):
from botocore.waiter import WaiterModel
model = WaiterModel({
'version': 2,
'waiters': {
waiter_id: {
'delay':
10,
'operation':
'DescribeJobQueues',
'maxAttempts':
20,
'acceptors': [{
'expected': 'VALID',
'matcher': 'pathAll',
'state': 'success',
'argument': 'jobQueues[].status'
}, {
'expected': 'INVALID',
'matcher': 'pathAny',
'state': 'failure',
'argument': 'jobQueues[].status'
}]
}
}
})
from botocore import waiter
return waiter.create_waiter_with_client(waiter_id, model,
self.batch_client)
def get_compute_job_waiter(self, waiter_id):
from botocore.waiter import WaiterModel, create_waiter_with_client
model = WaiterModel({
'version': 2,
'waiters': {
waiter_id: {
'delay':
60,
'operation':
'DescribeJobs',
'maxAttempts':
24 * 60 * 2, # timeout of 2 days
'acceptors': [{
'expected': 'SUCCEEDED',
'matcher': 'pathAll',
'state': 'success',
'argument': 'jobs[].status'
}, {
'expected': 'FAILED',
'matcher': 'pathAny',
'state': 'failure',
'argument': 'jobs[].status'
}]
}
}
})
return create_waiter_with_client(waiter_id, model, self.batch_client)
def __init__(self,
profile_name='default',
waiter_delay=1,
max_attempts=30,):
"""Initializes logging, the IAM client and waiter.
Args:
profile_name (str): defines the profile boto3 should use to authenticate,
defaults to 'default'
waiter_delay (int): defines the amount of seconds to wait before checking
if the report ran
max_attempts (int): defines the maximum number of times to check if the
report finished
"""
# Initialize private class variables
self.__waiter_delay = waiter_delay
self.__max_attempts = max_attempts
self.__jobs = []
self.__json = {}
self.__policy_arns = []
# Initialize IAM client
self.__initialize__iam_client(profile_name)
# Setup waiter for last access job
self.__waiter_model = WaiterModel(
last_access_details_waiter_config(waiter_delay, max_attempts))
self.__last_access_job_waiter = create_waiter_with_client(
LAST_ACCESS_DETAILS_WAITER_NAME, self.__waiter_model,
self.__iam_client)
def get_job_queue_waiter(self):
waiter_id = 'JobQueueWaiter'
model = WaiterModel({
'version': 2,
'waiters': {
waiter_id: {
'delay': 1,
'operation': 'DescribeJobQueues',
'maxAttempts': 10,
'acceptors': [
{
'expected': 'VALID',
'matcher': 'pathAll',
'state': 'success',
'argument': 'jobQueues[].status'
},
{
'expected': 'INVALID',
'matcher': 'pathAny',
'state': 'failure',
'argument': 'jobQueues[].status'
}
]
}
}
})
return create_waiter_with_client(waiter_id, model, self.client)
def setUp(self):
super(TestWaiterDocumenter, self).setUp()
self.add_shape_to_params('Biz', 'String')
self.setup_client()
waiter_model = WaiterModel(self.waiter_json_model)
self.waiter_documenter = WaiterDocumenter(
client=self.client, service_waiter_model=waiter_model)
def get_waiter_model(self, service, api_version=None):
"""Get the waiter model for the service."""
with mock.patch('botocore.loaders.Loader.list_available_services',
return_value=[service]):
return WaiterModel(
self.loader.load_service_model(service,
type_name='waiters-2',
api_version=api_version))
def test_get_waiter_does_not_exist(self):
waiters = {
'version': 2,
'waiters': {}
}
model = WaiterModel(waiters)
with self.assertRaises(ValueError):
model.get_waiter('UnknownWaiter')
def test_wont_load_missing_version_in_config(self):
# We only load waiter configs if we know for sure that we're
# loading version 2 of the format.
waiters = {
# Missing the 'version' key.
'waiters': {}
}
with self.assertRaises(WaiterConfigError):
WaiterModel(waiters)
def test_waiter_names(self):
waiters = {
'version': 2,
'waiters': {
'BarWaiter': {},
'FooWaiter': {},
}
}
self.assertEqual(
WaiterModel(waiters).waiter_names, ['BarWaiter', 'FooWaiter'])
def test_add_waiters_no_waiter_names(self):
self.session.get_waiter_model.return_value = WaiterModel({
'version': 2,
# No waiters are specified.
'waiters': {}
})
command_table = {}
add_waiters(command_table, self.session, self.command_object)
# Make sure that no wait command was added since the service object
# has no waiters.
self.assertEqual(command_table, {})
def get_waiter_model(self, service, api_version=None):
"""
Get the waiter model for the service
"""
service = os.path.join('aws', service)
model_version = self.loader.determine_latest(service, api_version)
# Some wierd formatting required to get the name of the model
# correct. Right now this is returned: YYYY-MM-DD.api
# We need: YYYY-MM-DD
model_version = ''.join(model_version.split('.')[:-1])
waiter_model = model_version + '.waiters'
return WaiterModel(self.loader.load_data(waiter_model))
def setUp(self):
self.model = WaiterModel({
'version': 2,
'waiters': {
'Foo': {
'operation': 'foo', 'maxAttempts': 1, 'delay': 1,
'acceptors': [],
}
}
})
self.service_object = mock.Mock()
self.cmd = WaitCommand(self.model, self.service_object)
def setUp(self):
self.waiter_config = {
'version': 2,
'waiters': {
'WaiterName': {
'operation': 'Foo',
'delay': 1,
'maxAttempts': 1,
'acceptors': [],
},
},
}
self.waiter_model = WaiterModel(self.waiter_config)
def setUp(self):
self.service_model = mock.Mock()
self.session = mock.Mock()
self.command_object = mock.Mock()
self.command_object.service_model = self.service_model
# Set up the mock session.
self.session.get_waiter_model.return_value = WaiterModel({
'version': 2,
'waiters': {
'FooExists': {},
}
})
def tgw_attachment_waiter(self, desired_state: AttachmentState,
attachment_id: str) -> None:
"""[summary]
Args:
desired_state (str): desired state of the tgw attachment
attachment_id (str): attachment-id
"""
delay = 10
max_attempts = 15
waiter_name = "TGWAttachmentInPendingAcceptance"
waiter_config = {
"version": 2,
"waiters": {
"TGWAttachmentInPendingAcceptance": {
"operation":
"DescribeTransitGatewayPeeringAttachments",
"delay":
delay,
"maxAttempts":
max_attempts,
"acceptors": [
{
"matcher": "path",
"expected": desired_state.value,
"argument":
"TransitGatewayPeeringAttachments[0].State",
"state": "success",
},
{
"matcher": "path",
"expected": AttachmentState.FAILED.value,
"argument":
"TransitGatewayPeeringAttachments[0].State",
"state": "failure",
},
],
}
},
}
waiter_model = WaiterModel(waiter_config)
custom_waiter = create_waiter_with_client(waiter_name, waiter_model,
self.ec2_client)
try:
custom_waiter.wait(TransitGatewayAttachmentIds=[attachment_id])
except WaiterError as err:
self.logger.error(str(err))
raise
def _load_prefect_waiter(boto_client: "boto3.client", client_str: str,
waiter_name: str):
"""
Load a custom waiter from the ./waiters directory.
"""
try:
# Instantiate waiter from accompanying client json file
with pkg_resources.open_text(waiters,
f"{client_str}.json") as handle:
waiter_model = WaiterModel(json.load(handle))
return create_waiter_with_client(waiter_name, waiter_model,
boto_client)
except Exception as err:
raise ValueError(
f"Unable to load waiter '{waiter_name}' for AWS client '{client_str}'."
) from err
def setUp(self):
self.waiter_config = {
'version': 2,
'waiters': {
'WaiterName': {
'operation': 'Foo',
'delay': 1,
'maxAttempts': 1,
'acceptors': [],
},
},
}
self.waiter_model = WaiterModel(self.waiter_config)
self.service_json_model = {
'metadata': {
'serviceFullName': 'Amazon MyService'
},
'operations': {
'Foo': {
'name': 'Foo',
'input': {
'shape': 'FooInputOutput'
},
'output': {
'shape': 'FooInputOutput'
}
}
},
'shapes': {
'FooInputOutput': {
'type': 'structure',
'members': {
'bar': {
'shape': 'String',
'documentation': 'Documents bar'
}
}
},
'String': {
'type': 'string'
}
}
}
self.service_model = ServiceModel(self.service_json_model, 'myservice')
self.client = mock.Mock()
self.client.meta.service_model = self.service_model
def gen_available_waiter(WAITER_ID, operation, delay, maxAttempts, path):
return WaiterModel({
'version': 2,
'waiters': {
WAITER_ID: {
'operation':
operation,
'delay':
delay,
'maxAttempts':
maxAttempts,
'acceptors': [{
'state': 'success',
'matcher': 'path',
'argument': f"{path} == 'available'",
'expected': True
}]
}
}
})
def delete_fargate_profile(
profile_name: str,
cluster_name: str,
) -> None:
_logger.debug(f"Deleting EKS Fargate Profile: {profile_name}")
if describe_fargate_profile(profile_name=profile_name,
cluster_name=cluster_name) is None:
_logger.debug(f"EKS Fargate Profile not found: {profile_name}")
return
eks_client = boto3_client("eks")
eks_client.delete_fargate_profile(
fargateProfileName=profile_name,
clusterName=cluster_name,
)
waiter_model = WaiterModel(WAITER_CONFIG)
waiter = create_waiter_with_client("FargateProfileDeleted", waiter_model,
eks_client)
waiter.wait(fargateProfileName=profile_name, clusterName=cluster_name)
_logger.debug(f"Deleted EKS Fargate Profile: {profile_name}")
def handle_waiters(client, client_name, class_name, service_name, service_path,
sidebar_lines):
waiter_config = client._get_waiter_config()
waiter_model = WaiterModel(
waiter_config) if 'waiters' in waiter_config else None
if not waiter_model:
return
waiters_path = f'{service_path}/waiters'
sidebar_lines.append(f' - [Waiters]({waiters_path})')
docs_waiters_path = f'docs/{waiters_path}.md'
waiter_names = waiter_model.waiter_names
example_waiter_name = waiter_names[0]
waiter_list_items = create_waiter_index(docs_waiters_path, client_name,
service_name, example_waiter_name)
for name in waiter_names:
handle_waiter(class_name, client_name, name, service_path,
waiter_list_items, waiter_model, waiters_path)
write_lines(docs_waiters_path, waiter_list_items)
def _create_delete_traffic_session_waiter(ec2_client):
delete_session_model = WaiterModel({
"version": 2,
"waiters": {
"TrafficMirrorDeleted": {
"delay":
15,
"operation":
"DescribeTrafficMirrorSessions",
"maxAttempts":
40,
"acceptors": [{
"matcher": "error",
"expected": "InvalidTrafficMirrorSessionId.NotFound",
"state": "success",
}],
}
},
})
delete_session_waiter = create_waiter_with_client(
"TrafficMirrorDeleted", delete_session_model, ec2_client)
return delete_session_waiter
def get_class_output(client_name):
method_signatures = []
shapes_in_classes = []
client = boto3.client(client_name)
class_name = type(client).__name__
service_model = client._service_model
waiter_config = client._get_waiter_config()
waiter_model = WaiterModel(
waiter_config) if 'waiters' in waiter_config else None
try:
paginator_model = botocore.session.get_session().get_paginator_model(
client_name)
except botocore.exceptions.UnknownServiceError:
paginator_model = None # meaning it probably doesn't have paginators
for name in service_model.operation_names:
method_signatures.append(
get_method_signature(service_model, name, shapes_in_classes,
class_name))
return get_class_signature(client_name, class_name,
service_model.documentation, method_signatures,
shapes_in_classes, waiter_model,
paginator_model)
def create_environment(self, auth_credentials: Dict, user: User,
environment: Environment):
org_client = self._get_client("organizations")
# Create an account. Requires organizations:CreateAccount permission
account_request = org_client.create_account(
Email=user.email,
AccountName=uuid4().hex,
IamUserAccessToBilling="ALLOW")
# Configuration for our CreateAccount Waiter.
# A waiter is a boto3 helper which can be configured to poll a given status
# endpoint until it succeeds or fails. boto3 has many built in waiters, but none
# for the organizations service so we're building our own here.
waiter_config = {
"version": 2,
"waiters": {
"AccountCreated": {
"operation":
"DescribeCreateAccountStatus",
"delay":
20,
"maxAttempts":
self.MAX_CREATE_ACCOUNT_ATTEMPTS,
"acceptors": [
{
"matcher": "path",
"expected": "SUCCEEDED",
"argument": "CreateAccountStatus.State",
"state": "success",
},
{
"matcher": "path",
"expected": "IN_PROGRESS",
"argument": "CreateAccountStatus.State",
"state": "retry",
},
{
"matcher": "path",
"expected": "FAILED",
"argument": "CreateAccountStatus.State",
"state": "failure",
},
],
}
},
}
waiter_model = WaiterModel(waiter_config)
account_waiter = create_waiter_with_client("AccountCreated",
waiter_model, org_client)
try:
# Poll until the CreateAccount request either succeeds or fails.
account_waiter.wait(
CreateAccountRequestId=account_request["CreateAccountStatus"]
["Id"])
except WaiterError:
# TODO: Possible failure reasons:
# 'ACCOUNT_LIMIT_EXCEEDED'|'EMAIL_ALREADY_EXISTS'|'INVALID_ADDRESS'|'INVALID_EMAIL'|'CONCURRENT_ACCOUNT_MODIFICATION'|'INTERNAL_FAILURE'
raise EnvironmentCreationException(environment.id,
"Failed to create account.")
# We need to re-fetch this since the Waiter throws away the success response for some reason.
created_account_status = org_client.describe_create_account_status(
CreateAccountRequestId=account_request["CreateAccountStatus"]
["Id"])
account_id = created_account_status["CreateAccountStatus"]["AccountId"]
return account_id
