def _get_execution_output(self, arn):
sfns = boto3.client('stepfunctions')
execution = sfns.describe_execution(executionArn=arn)
status = execution['status']
require(status in ('SUCCEEDED', 'RUNNING'), f'Unexpected execution status: {status}')
self.assertIn('output', execution)
return execution['output']
def __init__(self,
deployment_stage: str,
delays: List[float],
num_workers: int = None,
sqs_polling_timeout: int = 20,
timeout: float = 10.0,
overhead: float = 30.0) -> None:
"""
Create a new notifier.
:param deployment_stage: the name of the deployment stage that hosts the cloud resources used by this
notifier. Multiple notifier instances created with the same value for this parameter will share those
resources.
:param delays: a list of delays in seconds before notification attempts. There will be one SQS queue and
typically one worker thread per entry in this list. The first value is typically zero, unless the first
attempt should be delayed as well. Multiple notifier instances created with the same value for the
`deployment_stage` parameter should have the same value for the delays parameter. If they don't,
the resulting behavior is unspecififed.
:param num_workers: the number of notification workers pulling notifications from the queues and delivering
them. The default is to have one worker per queue. If the configured number of workers is less than the
number of queues, the notifier will assign the workers to a weighted random choice among the queues,
prioritizing longer queues. If the configured number of workers is greater than the number of queues,
the notifier will first ensure that every queue is assigned one worker and then assign the remaining workers
to a weighted random choice of queues, again prioritizing longer ones. Multiple notifier instances,
each with their own set of workers, can be pulling from the same set of queues. If the number of workers is
less than the number of queues, multiple notifier instances are needed to ensure that every queue is served.
:param sqs_polling_timeout: the integral number of seconds that a SQS ReceiveMessages request waits on any
queue before returning
:param timeout: the notification timeout in seconds. A notification attempt will be considered a failure if
the HTTP/HTTPS request to the corresponding notification endpoint takes longer than the specified amount of
time.
:param overhead: The expected amount of time needed for ancillary work around one notification attempt. This
does not include the time for making the actual HTTP(S) notification request but includes dequeueing and
potentially requeueing the notification. If this value is too low, duplicate notification attempts will be
made. If it is too high, throughput might be slightly impaired on the happy path, and considerably impaired
when workers (not the endpoints) fail and messages have to be requed by SQS VisibilityTimeout mechanism.
"""
super().__init__()
require(len(delays) > 0, "Must specify at least one delay value.")
require(num_workers is None or num_workers > 0, "If specified, the number of workers must be greater than 0.")
self._deployment_stage = deployment_stage
self._delays = delays
self._num_workers = len(delays) if num_workers is None else num_workers
self._sqs_polling_timeout = sqs_polling_timeout
self._timeout = timeout
self._overhead = overhead
def create(cls,
notification_id: str,
subscription_id: str,
url: str,
method: str,
encoding: str,
body: JSON,
attempts: Optional[int] = None,
hmac_key: Optional[bytes] = None,
hmac_key_id: Optional[str] = None,
correlation_id: Optional[str] = None) -> 'Notification':
allowed_schemes = {'https'} if DeploymentStage.IS_PROD() else {
'https', 'http'
}
scheme = urlparse(url).scheme
require(
scheme in allowed_schemes,
f"The scheme '{scheme}' of URL '{url}' is prohibited. Allowed schemes are {allowed_schemes}."
)
if DeploymentStage.IS_PROD():
hostname = urlparse(url).hostname
for family, socktype, proto, canonname, sockaddr in socket.getaddrinfo(
hostname, port=None):
require(
ipaddress.ip_address(sockaddr[0]).is_global,
f"The hostname in URL '{url}' resolves to a private IP")
if attempts is None:
attempts = Config.notification_attempts()
return cls(
notification_id=notification_id,
subscription_id=subscription_id,
url=url,
method=method,
encoding=encoding,
body=cls._bin2sqs(json.dumps(body).encode()),
attempts=attempts,
hmac_key=None if hmac_key is None else cls._bin2sqs(hmac_key),
hmac_key_id=hmac_key_id,
queued_at=
None, # this field will be set when the message comes out of the queue
correlation_id=correlation_id)
def enqueue(self, notification: Notification, queue_index: int = 0) -> None:
require(notification.attempts is not None,
"Cannot enqueue a notification whose `attempts` attribute is None", notification)
if queue_index is None:
# Enqueueing a notification into the failure queue does not consume an attempt.
logger.info(f"Adding notification to '{self._queue_name(None)}' queue as requested: {notification}")
elif notification.attempts > 0:
queues_left = self.num_queues - queue_index
if notification.attempts < queues_left:
logger.info(f"Notification would expire before proceeding through the remaining queues (%i): %s",
queues_left, notification)
# Enqueueing a notification into one of the regular queues consumes one attempt.
notification = notification.spend_attempt()
logger.info(f"Adding notification to queue {queue_index}: {notification}")
else:
logger.warning(f"Notification has no attempts left, giving up. Adding it to '{self._queue_name(None)}' "
f"instead of ({self._queue_name(queue_index)}) as originally requested: {notification}")
queue_index = None
queue = self._queue(queue_index)
queue.send_message(**notification.to_sqs_message())
def __init__(self, amount: float) -> None:
require(amount >= 0, "Inital remaining time must be non-negative")
super().__init__(time.time() + amount)