def create_waiter(evb_client):
waiter_name = "EvbRuleEnabled"
waiter_config = {
"version": 2,
"waiters": {
"EvbRuleEnabled": {
"operation":
"DescribeRule",
"delay":
WAITER_DELAY,
"maxAttempts":
WAITER_MAX_ATTEMPTS,
"acceptors": [
{
"matcher": "path",
"expected": "ENABLED",
"state": "success",
"argument": "State",
},
{
"matcher": "path",
"expected": "DISABLED",
"state": "retry",
"argument": "State",
},
],
}
},
}
waiter_model = WaiterModel(waiter_config)
return create_waiter_with_client(waiter_name, waiter_model, evb_client)
def wait_for_changes_in_aws(autoscaling_client, group_name, desired_capacity) -> None:
waiter_name = 'autoscaling_completed'
argument = f"contains(AutoScalingGroups[?(starts_with(AutoScalingGroupName, `{group_name}`) == `true`)]." \
f"[length(Instances[?LifecycleState=='InService']) == `{desired_capacity}`][], `false`)"
waiter_config = {
"version": 2,
"waiters": {
"autoscaling_completed": {
"acceptors": [
{
"argument": argument,
"expected": True,
"matcher": "path",
"state": "success"
},
{
"argument": argument,
"expected": False,
"matcher": "path",
"state": "retry"
}
],
"delay": 5,
"maxAttempts": 20,
"operation": "DescribeAutoScalingGroups"
}
}
}
waiter_model = WaiterModel(waiter_config)
custom_waiter = create_waiter_with_client(waiter_name, waiter_model, autoscaling_client)
custom_waiter.wait()
def get_job_queue_waiter(self):
waiter_id = 'JobQueueWaiter'
model = WaiterModel({
'version': 2,
'waiters': {
waiter_id: {
'delay': 1,
'operation': 'DescribeJobQueues',
'maxAttempts': 10,
'acceptors': [
{
'expected': 'VALID',
'matcher': 'pathAll',
'state': 'success',
'argument': 'jobQueues[].status'
},
{
'expected': 'INVALID',
'matcher': 'pathAny',
'state': 'failure',
'argument': 'jobQueues[].status'
}
]
}
}
})
return create_waiter_with_client(waiter_id, model, self.client)
def get_compute_job_waiter(self, waiter_id):
from botocore.waiter import WaiterModel, create_waiter_with_client
model = WaiterModel({
'version': 2,
'waiters': {
waiter_id: {
'delay':
60,
'operation':
'DescribeJobs',
'maxAttempts':
24 * 60 * 2, # timeout of 2 days
'acceptors': [{
'expected': 'SUCCEEDED',
'matcher': 'pathAll',
'state': 'success',
'argument': 'jobs[].status'
}, {
'expected': 'FAILED',
'matcher': 'pathAny',
'state': 'failure',
'argument': 'jobs[].status'
}]
}
}
})
return create_waiter_with_client(waiter_id, model, self.batch_client)
def stop_scheduler(ctx, wait, yes):
"""Stop the scheduler service.
Only one Airflow scheduler can be running at a time. During a new
service redeploy through Terraform this would result in a brief period
where two schedulers are running simultaneously. The process for
deploying a new service should be to use this command to stop the
scheduler, then deploy the new service through Terraform, then run the
start-scheduler command.
"""
cluster = ctx.obj['cluster']
if not yes:
selected = click.style(cluster.name, fg='yellow', bold=True)
click.confirm(
f'Are you sure you want to stop the scheduler in the {selected} '
'cluster?',
abort=True)
stop_waiter = boto_waiter.create_waiter_with_client(
'ServiceDrained', ecs_model, cluster.ecs)
with check_task():
cluster.stop(services=[cluster.scheduler])
click.echo('Stopping scheduler...', nl=False)
if not wait:
return
stop_waiter.wait(cluster=cluster.name, services=[cluster.scheduler])
click.echo("\033[1A")
ok = click.style('OK', fg='green')
click.echo(f'Stopping scheduler...{ok}')
def __init__(self,
profile_name='default',
waiter_delay=1,
max_attempts=30,):
"""Initializes logging, the IAM client and waiter.
Args:
profile_name (str): defines the profile boto3 should use to authenticate,
defaults to 'default'
waiter_delay (int): defines the amount of seconds to wait before checking
if the report ran
max_attempts (int): defines the maximum number of times to check if the
report finished
"""
# Initialize private class variables
self.__waiter_delay = waiter_delay
self.__max_attempts = max_attempts
self.__jobs = []
self.__json = {}
self.__policy_arns = []
# Initialize IAM client
self.__initialize__iam_client(profile_name)
# Setup waiter for last access job
self.__waiter_model = WaiterModel(
last_access_details_waiter_config(waiter_delay, max_attempts))
self.__last_access_job_waiter = create_waiter_with_client(
LAST_ACCESS_DETAILS_WAITER_NAME, self.__waiter_model,
self.__iam_client)
def fargate_delete_waiter(self, client):
# Fargate profiles seem to delete faster @ roughly 2 minutes each so keeping defaults
config = {
'version': 2,
'waiters': {
"FargateProfileDeleted": {
'operation':
'DescribeFargateProfile',
'delay':
30,
'maxAttempts':
40,
'acceptors': [{
"expected": "DELETE_FAILED",
"matcher": "path",
"state": "failure",
"argument": "fargateprofile.status"
}, {
"expected": "ResourceNotFoundException",
"matcher": "error",
"state": "success"
}]
}
}
}
return create_waiter_with_client("FargateProfileDeleted",
WaiterModel(config), client)
def create_fargate_profile(
profile_name: str,
cluster_name: str,
role_arn: str,
subnets: List[str],
namespace: str,
selector_labels: Optional[Dict[str, Any]] = None,
) -> None:
_logger.debug(f"Creating EKS Fargate Profile: {profile_name}")
if describe_fargate_profile(profile_name=profile_name,
cluster_name=cluster_name) is not None:
_logger.debug(f"EKS Fargate Profile already exists: {profile_name}")
return
eks_client = boto3_client("eks")
eks_client.create_fargate_profile(
fargateProfileName=profile_name,
clusterName=cluster_name,
podExecutionRoleArn=role_arn,
subnets=subnets,
selectors=[{
"namespace": namespace,
"labels": selector_labels
}],
)
waiter_model = WaiterModel(WAITER_CONFIG)
waiter = create_waiter_with_client("FargateProfileCreated", waiter_model,
eks_client)
waiter.wait(fargateProfileName=profile_name, clusterName=cluster_name)
_logger.debug(f"Created EKS Fargate Profile: {profile_name}")
def get_compute_job_queue_waiter(self, waiter_id):
from botocore.waiter import WaiterModel
model = WaiterModel({
'version': 2,
'waiters': {
waiter_id: {
'delay':
10,
'operation':
'DescribeJobQueues',
'maxAttempts':
20,
'acceptors': [{
'expected': 'VALID',
'matcher': 'pathAll',
'state': 'success',
'argument': 'jobQueues[].status'
}, {
'expected': 'INVALID',
'matcher': 'pathAny',
'state': 'failure',
'argument': 'jobQueues[].status'
}]
}
}
})
from botocore import waiter
return waiter.create_waiter_with_client(waiter_id, model,
self.batch_client)
def test_waiter_class_name(self):
waiter_name = 'WaiterName'
waiter = create_waiter_with_client(
waiter_name, self.waiter_model, self.client)
self.assertEqual(
waiter.__class__.__name__,
'MyService.Waiter.WaiterName'
)
def wait_cluster_available(neptune, cluster_id, delay, maxAttempts):
WAITER_ID = 'neptune_cluster_available'
WAITER_MODEL = gen_available_waiter(WAITER_ID, 'DescribeDBClusters', delay,
maxAttempts, 'DBClusters[0].Status')
waiter = create_waiter_with_client(WAITER_ID, WAITER_MODEL, neptune)
waiter.wait(DBClusterIdentifier=cluster_id)
def test_waiter_class_name(self):
waiter_name = 'WaiterName'
waiter = create_waiter_with_client(
waiter_name, self.waiter_model, self.client)
self.assertEqual(
waiter.__class__.__name__,
'MyService.Waiter.WaiterName'
)
def __create(self, name: str, model: waiter.WaiterModel,
client: BotoClient):
if name not in model.waiter_names:
raise self.message_formatter.get_error(KeyError,
'Waiter %s does not exist',
name)
self.waiters.update(
{name: waiter.create_waiter_with_client(name, model, client)})
def wait_db_instance_available(neptune, clone_instance_id, delay, maxAttempts):
WAITER_ID = 'neptune_db_instance_available'
WAITER_MODEL = gen_available_waiter(WAITER_ID, 'DescribeDBInstances',
delay, maxAttempts,
'DBInstances[0].DBInstanceStatus')
waiter = create_waiter_with_client(WAITER_ID, WAITER_MODEL, neptune)
waiter.wait(DBInstanceIdentifier=clone_instance_id)
def assert_distribution_deployed_call_count(self, api_version=None):
waiter_name = 'DistributionDeployed'
waiter_model = self.get_waiter_model(self.service, api_version)
self.client.get_distribution.side_effect = [
{'Distribution': {'Status': 'Deployed'}}
]
waiter = create_waiter_with_client(waiter_name, waiter_model,
self.client)
waiter.wait()
self.assertEqual(self.client.get_distribution.call_count, 1)
def assert_invalidation_completed_call_count(self, api_version=None):
waiter_name = 'InvalidationCompleted'
waiter_model = self.get_waiter_model(self.service, api_version)
self.client.get_invalidation.side_effect = [
{'Invalidation': {'Status': 'Completed'}}
]
waiter = create_waiter_with_client(waiter_name, waiter_model,
self.client)
waiter.wait()
self.assertEqual(self.client.get_invalidation.call_count, 1)
def redeploy(ctx, yes):
"""Redeploy Airflow cluster.
This will perform a full redeploy of the Airflow cluster. The cluster
has to be brought down and back up in a specific order or otherwise
race conditions could arise. First, the scheduler is stopped. Then,
the web and worker services are redeployed. Finally, the scheduler is
started back up with the new deployment.
*IMPORTANT* Always use this command to redeploy as it will ensure the
system remains in a consistent state. It should be safe to run again
even if it has previously failed partway through due to, for example,
a network failure.
It will take several minutes for the cluster to fully restart. Be patient.
\b
Example usage:
$ workflow --cluster airflow-prod redeploy
"""
cluster = ctx.obj['cluster']
if not yes:
selected = click.style(cluster.name, fg='yellow', bold=True)
click.confirm(
f'Are you sure you want to redeploy the {selected} cluster?',
abort=True)
stop_waiter = boto_waiter.create_waiter_with_client(
'ServiceDrained', ecs_model, cluster.ecs)
start_waiter = cluster.ecs.get_waiter('services_stable')
with check_task():
cluster.stop(services=[cluster.scheduler])
click.echo('Stopping scheduler.......', nl=False)
stop_waiter.wait(cluster=cluster.name, services=[cluster.scheduler])
click.echo("\033[1A")
ok = click.style('OK', fg='green')
click.echo(f'Stopping scheduler.......{ok}')
with check_task():
cluster.start(services=[cluster.worker, cluster.web])
click.echo('Redeploying web/worker...', nl=False)
start_waiter.wait(cluster=cluster.name,
services=[cluster.web, cluster.worker])
click.echo("\033[1A")
ok = click.style('OK', fg='green')
click.echo(f'Redeploying web/worker...{ok}')
with check_task():
cluster.start(services=[cluster.scheduler])
click.echo('Starting scheduler.......', nl=False)
start_waiter.wait(cluster=cluster.name, services=[cluster.scheduler])
click.echo("\033[1A")
ok = click.style('OK', fg='green')
click.echo(f'Starting scheduler.......{ok}')
def assert_distribution_deployed_call_count(self, api_version=None):
waiter_name = 'DistributionDeployed'
waiter_model = self.get_waiter_model(self.service, api_version)
self.client.get_distribution.side_effect = [{
'Distribution': {
'Status': 'Deployed'
}
}]
waiter = create_waiter_with_client(waiter_name, waiter_model,
self.client)
waiter.wait()
self.assertEqual(self.client.get_distribution.call_count, 1)
def get_waiter(self, waiter_name):
config = self._get_waiter_config()
if not config:
raise ValueError("Waiter does not exist: %s" % waiter_name)
model = waiter.WaiterModel(config)
mapping = {}
for name in model.waiter_names:
mapping[xform_name(name)] = name
if waiter_name not in mapping:
raise ValueError("Waiter does not exist: %s" % waiter_name)
return waiter.create_waiter_with_client(mapping[waiter_name], model, self)
def assert_invalidation_completed_call_count(self, api_version=None):
waiter_name = 'InvalidationCompleted'
waiter_model = self.get_waiter_model(self.service, api_version)
self.client.get_invalidation.side_effect = [{
'Invalidation': {
'Status': 'Completed'
}
}]
waiter = create_waiter_with_client(waiter_name, waiter_model,
self.client)
waiter.wait()
self.assertEqual(self.client.get_invalidation.call_count, 1)
def get_waiter(self, waiter_name):
config = self._get_waiter_config()
if not config:
raise ValueError("Waiter does not exist: %s" % waiter_name)
model = waiter.WaiterModel(config)
mapping = {}
for name in model.waiter_names:
mapping[xform_name(name)] = name
if waiter_name not in mapping:
raise ValueError("Waiter does not exist: %s" % waiter_name)
return waiter.create_waiter_with_client(mapping[waiter_name], model,
self)
def tgw_attachment_waiter(self, desired_state: AttachmentState,
attachment_id: str) -> None:
"""[summary]
Args:
desired_state (str): desired state of the tgw attachment
attachment_id (str): attachment-id
"""
delay = 10
max_attempts = 15
waiter_name = "TGWAttachmentInPendingAcceptance"
waiter_config = {
"version": 2,
"waiters": {
"TGWAttachmentInPendingAcceptance": {
"operation":
"DescribeTransitGatewayPeeringAttachments",
"delay":
delay,
"maxAttempts":
max_attempts,
"acceptors": [
{
"matcher": "path",
"expected": desired_state.value,
"argument":
"TransitGatewayPeeringAttachments[0].State",
"state": "success",
},
{
"matcher": "path",
"expected": AttachmentState.FAILED.value,
"argument":
"TransitGatewayPeeringAttachments[0].State",
"state": "failure",
},
],
}
},
}
waiter_model = WaiterModel(waiter_config)
custom_waiter = create_waiter_with_client(waiter_name, waiter_model,
self.ec2_client)
try:
custom_waiter.wait(TransitGatewayAttachmentIds=[attachment_id])
except WaiterError as err:
self.logger.error(str(err))
raise
def _load_prefect_waiter(boto_client: "boto3.client", client_str: str,
waiter_name: str):
"""
Load a custom waiter from the ./waiters directory.
"""
try:
# Instantiate waiter from accompanying client json file
with pkg_resources.open_text(waiters,
f"{client_str}.json") as handle:
waiter_model = WaiterModel(json.load(handle))
return create_waiter_with_client(waiter_name, waiter_model,
boto_client)
except Exception as err:
raise ValueError(
f"Unable to load waiter '{waiter_name}' for AWS client '{client_str}'."
) from err
def _create_delete_traffic_session_waiter(ec2_client):
delete_session_model = WaiterModel({
"version": 2,
"waiters": {
"TrafficMirrorDeleted": {
"delay":
15,
"operation":
"DescribeTrafficMirrorSessions",
"maxAttempts":
40,
"acceptors": [{
"matcher": "error",
"expected": "InvalidTrafficMirrorSessionId.NotFound",
"state": "success",
}],
}
},
})
delete_session_waiter = create_waiter_with_client(
"TrafficMirrorDeleted", delete_session_model, ec2_client)
return delete_session_waiter
def test_waiter_help_documentation(self):
waiter_name = 'WaiterName'
waiter = create_waiter_with_client(waiter_name, self.waiter_model,
self.client)
with mock.patch('sys.stdout', six.StringIO()) as mock_stdout:
help(waiter.wait)
content = mock_stdout.getvalue()
lines = [
(' Polls :py:meth:`MyService.Client.foo` every 1 '
'seconds until a successful state is reached. An error '
'is returned after 1 failed checks.'),
' **Request Syntax** ',
' ::',
' waiter.wait(',
" bar='string'",
' )',
' :type bar: string',
' :param bar: Documents bar',
' :returns: None',
]
for line in lines:
self.assertIn(line, content)
def delete_fargate_profile(
profile_name: str,
cluster_name: str,
) -> None:
_logger.debug(f"Deleting EKS Fargate Profile: {profile_name}")
if describe_fargate_profile(profile_name=profile_name,
cluster_name=cluster_name) is None:
_logger.debug(f"EKS Fargate Profile not found: {profile_name}")
return
eks_client = boto3_client("eks")
eks_client.delete_fargate_profile(
fargateProfileName=profile_name,
clusterName=cluster_name,
)
waiter_model = WaiterModel(WAITER_CONFIG)
waiter = create_waiter_with_client("FargateProfileDeleted", waiter_model,
eks_client)
waiter.wait(fargateProfileName=profile_name, clusterName=cluster_name)
_logger.debug(f"Deleted EKS Fargate Profile: {profile_name}")
def test_waiter_help_documentation(self):
waiter_name = 'WaiterName'
waiter = create_waiter_with_client(
waiter_name, self.waiter_model, self.client)
with mock.patch('sys.stdout', six.StringIO()) as mock_stdout:
help(waiter.wait)
content = mock_stdout.getvalue()
lines = [
(' Polls :py:meth:`MyService.Client.foo` every 1 '
'seconds until a successful state is reached. An error '
'is returned after 1 failed checks.'),
' **Request Syntax** ',
' ::',
' waiter.wait(',
" bar='string'",
' )',
' :type bar: string',
' :param bar: Documents bar',
' :returns: None',
]
for line in lines:
self.assertIn(line, content)
def get_waiter(self, waiter_name):
"""Returns an object that can wait for some condition.
:type waiter_name: str
:param waiter_name: The name of the waiter to get. See the waiters
section of the service docs for a list of available waiters.
:returns: The specified waiter object.
:rtype: botocore.waiter.Waiter
"""
config = self._get_waiter_config()
if not config:
raise ValueError("Waiter does not exist: %s" % waiter_name)
model = waiter.WaiterModel(config)
mapping = {}
for name in model.waiter_names:
mapping[xform_name(name)] = name
if waiter_name not in mapping:
raise ValueError("Waiter does not exist: %s" % waiter_name)
return waiter.create_waiter_with_client(mapping[waiter_name], model,
self)
def get_waiter(self, waiter_name):
"""Returns an object that can wait for some condition.
:type waiter_name: str
:param waiter_name: The name of the waiter to get. See the waiters
section of the service docs for a list of available waiters.
:returns: The specified waiter object.
:rtype: botocore.waiter.Waiter
"""
config = self._get_waiter_config()
if not config:
raise ValueError("Waiter does not exist: %s" % waiter_name)
model = waiter.WaiterModel(config)
mapping = {}
for name in model.waiter_names:
mapping[xform_name(name)] = name
if waiter_name not in mapping:
raise ValueError("Waiter does not exist: %s" % waiter_name)
return waiter.create_waiter_with_client(
mapping[waiter_name], model, self)
def __init__(self, client):
waiter_json_filename = os.path.join(utils.__path__[0], 'cfn-waiters-2.json')
with open(waiter_json_filename, 'r') as waiter_json_file:
self.waiter_json_model = json.load(waiter_json_file)
self.waiter_model = WaiterModel(self.waiter_json_model)
self.waiter = create_waiter_with_client('StackAvailable', self.waiter_model, client.meta.client)
def test_can_create_waiter_from_client(self):
waiter_name = 'WaiterName'
waiter = create_waiter_with_client(
waiter_name, self.waiter_model, self.client)
self.assertIsInstance(waiter, Waiter)
def lambda_handler(event, context):
# Set base parameters for each simulation job. Order from Lambda event, then environment variables.
if 'simulationJobParams' in event:
if 'vpcConfig' in event['simulationJobParams']:
sim_job_params['vpcConfig'] = sim_job_params['vpcConfig']
if 'iamRole' in event['simulationJobParams']:
sim_job_params['iamRole'] = sim_job_params['iamRole']
if 'outputLocation' in event['simulationJobParams']:
sim_job_params['outputLocation'] = sim_job_params['outputLocation']
if 'simulationApplicationArn' in event:
app_arn = event['simulationApplicationArn']
if 'serverIP' in event:
private_ip = event['serverIP']
else:
# Launch Server.
server_app_params = create_application_config(event['server'], True, 'localhost')
server_job_params = deepcopy(sim_job_params)
server_job_params['simulationApplications'].append(server_app_params)
server_job_response = robomaker.create_simulation_job(
iamRole=server_job_params["iamRole"],
maxJobDurationInSeconds=server_job_params["maxJobDurationInSeconds"],
simulationApplications=[server_app_params],
vpcConfig=server_job_params["vpcConfig"],
loggingConfig=server_job_params["loggingConfig"],
outputLocation=server_job_params["outputLocation"]
)
# Wait for server to be available.
waiter_name = 'SimJobCreated'
waiter_model = WaiterModel(waiter_config)
custom_waiter = create_waiter_with_client(waiter_name, waiter_model, robomaker)
custom_waiter.wait(job=server_job_response['arn'])
desc_result = robomaker.describe_simulation_job( job = server_job_response['arn'] )
private_ip = desc_result['networkInterface']['privateIpAddress']
# Launch multiple robot batches.
batch_job_requests = []
client_app_params = {}
client_job_params = {}
for robot in event['robots']:
client_app_params[robot['name']] = create_application_config(robot, False, private_ip)
client_job_params[robot['name']] = deepcopy(sim_job_params)
client_job_params[robot['name']]['simulationApplications'].append(client_app_params[robot['name']])
batch_job_requests.append(client_job_params[robot['name']])
response = robomaker.start_simulation_job_batch(
batchPolicy={
'timeoutInSeconds': DEFAULT_MAX_DURATION,
'maxConcurrency': len(event['robots'])
},
createSimulationJobRequests=batch_job_requests,
tags = {
'launcher': 'multi_robot_fleet'
})
return {
'statusCode': 200,
'body': response['arn']
}
def create_environment(self, auth_credentials: Dict, user: User,
environment: Environment):
org_client = self._get_client("organizations")
# Create an account. Requires organizations:CreateAccount permission
account_request = org_client.create_account(
Email=user.email,
AccountName=uuid4().hex,
IamUserAccessToBilling="ALLOW")
# Configuration for our CreateAccount Waiter.
# A waiter is a boto3 helper which can be configured to poll a given status
# endpoint until it succeeds or fails. boto3 has many built in waiters, but none
# for the organizations service so we're building our own here.
waiter_config = {
"version": 2,
"waiters": {
"AccountCreated": {
"operation":
"DescribeCreateAccountStatus",
"delay":
20,
"maxAttempts":
self.MAX_CREATE_ACCOUNT_ATTEMPTS,
"acceptors": [
{
"matcher": "path",
"expected": "SUCCEEDED",
"argument": "CreateAccountStatus.State",
"state": "success",
},
{
"matcher": "path",
"expected": "IN_PROGRESS",
"argument": "CreateAccountStatus.State",
"state": "retry",
},
{
"matcher": "path",
"expected": "FAILED",
"argument": "CreateAccountStatus.State",
"state": "failure",
},
],
}
},
}
waiter_model = WaiterModel(waiter_config)
account_waiter = create_waiter_with_client("AccountCreated",
waiter_model, org_client)
try:
# Poll until the CreateAccount request either succeeds or fails.
account_waiter.wait(
CreateAccountRequestId=account_request["CreateAccountStatus"]
["Id"])
except WaiterError:
# TODO: Possible failure reasons:
# 'ACCOUNT_LIMIT_EXCEEDED'|'EMAIL_ALREADY_EXISTS'|'INVALID_ADDRESS'|'INVALID_EMAIL'|'CONCURRENT_ACCOUNT_MODIFICATION'|'INTERNAL_FAILURE'
raise EnvironmentCreationException(environment.id,
"Failed to create account.")
# We need to re-fetch this since the Waiter throws away the success response for some reason.
created_account_status = org_client.describe_create_account_status(
CreateAccountRequestId=account_request["CreateAccountStatus"]
["Id"])
account_id = created_account_status["CreateAccountStatus"]["AccountId"]
return account_id
def execution_waiter(sf_client):
waiter_model = get_waiter_model("stepfunctions.json")
return create_waiter_with_client("ExecutionComplete", waiter_model,
sf_client)
def execution_exists_waiter(sf_client):
waiter_model = get_waiter_model("stepfunctions.json")
return create_waiter_with_client("ExecutionExists", waiter_model,
sf_client)
def job_complete_waiter(ddb_client):
waiter_model = get_waiter_model("jobs.json")
return create_waiter_with_client("JobComplete", waiter_model, ddb_client)
def job_finished_waiter(ddb_client):
waiter_model = get_waiter_model("jobs.json")
return create_waiter_with_client("JobFinished", waiter_model, ddb_client)
def job_exists_waiter(ddb_client):
waiter_model = get_waiter_model("jobs.json")
return create_waiter_with_client("JobExists", waiter_model, ddb_client)
