def notify(
expect: bool, # whether the message should make it
max_attempts: Optional[int] = None, # how many attempts to allow
responses: List[Tuple[
float,
int]] = None, # a list of (delay, http_status) tuples, one per attempt
attempts=None
): # expected number of attempts, currently only used to estimate the running time
if responses is None:
responses = [(0.0, 200)]
verify = random.random() > .5
notification_id = str(next(self.notification_id))
body: JSON = dict(notification_id=notification_id,
responses=responses,
verify=verify)
notification = Notification.create(
notification_id=notification_id,
subscription_id=str(random.choice(self.subscription_ids)),
url=f"http://{self.address}:{self.port}/{notification_id}",
method='POST',
encoding='application/json',
body=body,
attempts=max_attempts,
hmac_key=PostTestHandler.hmac_secret_key if verify else None,
hmac_key_id='1234' if verify else None)
nonlocal total_attempts
total_attempts += min(
notification.attempts,
self.num_queues) if attempts is None else attempts
(expected_receptions
if expect else expected_misses).add(notification_id)
self.notifier.enqueue(notification)
def enqueue(self, notification: Notification, queue_index: int = 0) -> None:
require(notification.attempts is not None,
"Cannot enqueue a notification whose `attempts` attribute is None", notification)
if queue_index is None:
# Enqueueing a notification into the failure queue does not consume an attempt.
logger.info(f"Adding notification to '{self._queue_name(None)}' queue as requested: {notification}")
elif notification.attempts > 0:
queues_left = self.num_queues - queue_index
if notification.attempts < queues_left:
logger.info(f"Notification would expire before proceeding through the remaining queues (%i): %s",
queues_left, notification)
# Enqueueing a notification into one of the regular queues consumes one attempt.
notification = notification.spend_attempt()
logger.info(f"Adding notification to queue {queue_index}: {notification}")
else:
logger.warning(f"Notification has no attempts left, giving up. Adding it to '{self._queue_name(None)}' "
f"instead of ({self._queue_name(queue_index)}) as originally requested: {notification}")
queue_index = None
queue = self._queue(queue_index)
queue.send_message(**notification.to_sqs_message())
def _notify_subscriber(self, doc: BundleDocument, subscription: dict):
transaction_id = str(uuid.uuid4())
subscription_id = subscription['id']
endpoint = Endpoint.from_subscription(subscription)
payload = dict(transaction_id=transaction_id,
subscription_id=subscription_id,
es_query=subscription['es_query'],
match=dict(bundle_uuid=doc.fqid.uuid,
bundle_version=doc.fqid.version))
definitions = subscription.get('attachments')
# Only mention attachments in the notification if the subscription does, too.
if definitions is not None:
payload['attachments'] = attachment.select(definitions, doc)
if endpoint.encoding == 'application/json':
body = payload
elif endpoint.encoding == 'multipart/form-data':
body = endpoint.form_fields.copy()
body[endpoint.payload_form_field] = json.dumps(payload)
else:
raise ValueError(f"Encoding {endpoint.encoding} is not supported")
try:
hmac_key = subscription['hmac_secret_key']
except KeyError:
hmac_key = None
hmac_key_id = None
else:
hmac_key = hmac_key.encode()
hmac_key_id = subscription.get('hmac_key_id', "hca-dss:" + subscription_id)
notification = Notification.create(notification_id=transaction_id,
subscription_id=subscription_id,
url=endpoint.callback_url,
method=endpoint.method,
encoding=endpoint.encoding,
body=body,
hmac_key=hmac_key,
hmac_key_id=hmac_key_id,
correlation_id=str(doc.fqid))
if self.notifier:
logger.info(f"Queueing asynchronous notification {notification} for bundle {doc.fqid}")
self.notifier.enqueue(notification)
else:
logger.info(f"Synchronously sending notification {notification} about bundle {doc.fqid}")
notification.deliver_or_raise()
def _worker(self, worker_index: int, queue_index: int, remaining_time: RemainingTime) -> None:
queue = self._queue(queue_index)
while self._timeout + self._sqs_polling_timeout < remaining_time.get():
visibility_timeout = self._timeout + self._overhead
messages = queue.receive_messages(MaxNumberOfMessages=1,
WaitTimeSeconds=self._sqs_polling_timeout,
AttributeNames=['All'],
MessageAttributeNames=['*'],
VisibilityTimeout=int(math.ceil(visibility_timeout)))
if messages:
assert len(messages) == 1
message = messages[0]
notification = Notification.from_sqs_message(message)
logger.info(f'Worker {worker_index} received message from queue {queue_index} for {notification}')
seconds_to_maturity = notification.queued_at + self._delays[queue_index] - time.time()
if seconds_to_maturity > 0:
# Hide the message and sleep until it matures. These two measures prevent immature messages from
# continuously bouncing between the queue and the workers consuming it, thereby preventing
# unnecessary churn on the queue. Consider that other messages further up in the queue
# invariantly mature after the current message, ensuring that this wait does not limit throughput
# or increase latency.
#
# TODO: determine how this interacts with FIFO queues and message groups as those yield
# messages in an ordering that, while being strict with respect to a group, is only partial
# with respect to the entire queue. The above invariant may not hold globally for that reason.
#
# SQS ignores a request to change the VTO of a message if the total VTO would exceed the max.
# allowed value of 12 hours. To be safe, we subtract the initial VTO from the max VTO.
max_visibility_timeout = SQS_MAX_VISIBILITY_TIMEOUT - visibility_timeout
visibility_timeout = min(seconds_to_maturity, max_visibility_timeout)
logger.info(f'Worker {worker_index} hiding message from queue {queue_index} '
f'for another {visibility_timeout:.3f}s. '
f'It will be {seconds_to_maturity:.3f}s to maturity of {notification}.')
message.change_visibility(VisibilityTimeout=int(visibility_timeout))
time.sleep(min(seconds_to_maturity, remaining_time.get()))
elif remaining_time.get() < visibility_timeout:
logger.info(f'Worker {worker_index} returning message to queue {queue_index}. '
f'There is not enough time left to deliver {notification}.')
message.change_visibility(VisibilityTimeout=0)
else:
if not notification.deliver(timeout=self._timeout, attempt=queue_index):
self.enqueue(notification, self._next_queue_index(queue_index))
message.delete()
else:
logger.info(f"Exiting worker {worker_index} due to insufficient time left.")