def handle_replay_sqs_failures(args: Any) -> None:
sqs = get_boto3_resource("sqs")
kinesis = get_boto3_client("kinesis")
queue_name = f"{args.sqs_queue}-failures-{args.stage}"
stream_name = f"{args.kinesis_stream}-{args.stage}"
queue = sqs.get_queue_by_name(QueueName=queue_name)
while True:
print(f"Getting messages from {queue_name}...")
messages = queue.receive_messages(WaitTimeSeconds=1)
if not messages:
print(f"Reached end of {queue_name}")
return
for message in messages:
print(f"Message ID: {message.message_id}")
print(json.dumps(json.loads(message.body), indent=4, sort_keys=True))
if args.print_only:
continue
response = input(
f"Re-publish message {message.message_id} to {stream_name}? (yes/no)\n"
).lower()
if response in ["y", "yes"]:
partition_key = input(f"What partition in {args.kinesis_stream}?\n").lower()
records = [{"Data": message.body, "PartitionKey": partition_key}]
print(
f"Re-publishing message {message.message_id} to {stream_name} at partition {partition_key}"
)
response = kinesis.put_records(StreamName=stream_name, Records=records)
print(f"Deleting message {message.message_id}\n")
message.delete()
else:
print(f"Skipping message {message.message_id}\n")
def handler(event: dict, context: dict) -> dict:
verify_deploy_stage()
kinesis = get_boto3_client("kinesis")
stage = os.environ["STAGE"]
idempotency_checker = IdempotencyChecker()
inbound_commands = [
_make_inbound_command(record) for record in event["Records"]
]
for command in inbound_commands:
if command.command_type is InboundCommandType.INBOUND_SMS:
if not idempotency_checker.already_processed(
command.sequence_number, IDEMPOTENCY_REALM):
twilio_webhook = command.payload["twilio_webhook"]
kinesis.put_record(
Data=json.dumps({
"type": "INBOUND_SMS",
"payload": twilio_webhook
}),
PartitionKey=command.payload["From"],
StreamName=f"message-log-{stage}",
)
idempotency_checker.record_as_processed(
command.sequence_number,
IDEMPOTENCY_REALM,
IDEMPOTENCY_EXPIRATION_MINUTES,
)
return {"statusCode": 200}
def _publish_commands(self, commands: List[Tuple[str, Dict[str,
Any]]]) -> None:
kinesis = get_boto3_client("kinesis")
records = [{
"Data": json.dumps(data),
"PartitionKey": phone_number
} for phone_number, data in commands]
response = kinesis.put_records(
StreamName=f"command-stream-{self.stage}", Records=records)
if response.get("FailedRecordCount"):
rollbar.report_exc_info(extra_data=response)
def get_all_users(args: Any) -> None:
dynamodb = get_boto3_client("dynamodb")
table_name = f"dialog-state-{args.stage}"
args = {}
while True:
result = dynamodb.scan(TableName=table_name, **args)
for item in result["Items"]:
print(item["phone_number"]["S"])
if not result.get("LastEvaluatedKey"):
break
args["ExclusiveStartKey"] = result["LastEvaluatedKey"]
def handle_show_stream_record(args: Any) -> None:
kinesis = get_boto3_client("kinesis")
stream_name = f"{args.kinesis_stream}-{args.stage}"
shard_iterator = kinesis.get_shard_iterator(
StreamName=stream_name,
ShardId=args.shard_id,
ShardIteratorType="AT_SEQUENCE_NUMBER",
StartingSequenceNumber=args.seq,
)
response = kinesis.get_records(ShardIterator=shard_iterator["ShardIterator"], Limit=1)
for record in response["Records"]:
print(json.loads(record["Data"]))
def _publish_event_batches_to_kinesis(event_batches: List[DialogEventBatch]) -> None:
kinesis = get_boto3_client("kinesis")
stage = os.environ.get("STAGE")
stream_name = f"dialog-event-batches-{stage}"
records = [
{
"PartitionKey": event_batch.phone_number,
"Data": event_batch.json(),
}
for event_batch in event_batches
]
response = kinesis.put_records(StreamName=stream_name, Records=records)
if response.get("FailedRecordCount"):
rollbar.report_exc_info(extra_data=response)
def publish_outbound_sms(payload: OutboundPayload) -> dict:
kinesis = get_boto3_client("kinesis")
stage = os.environ.get("STAGE")
records = [
{
"Data": json.dumps(
{
"type": "OUTBOUND_SMS",
"payload": payload.dict(),
}
),
"PartitionKey": payload.To,
}
]
return cast(dict, kinesis.put_records(Records=records, StreamName=f"message-log-{stage}"))
def _get_dialog_events(phone_number: str, stage: str) -> Iterator[DialogEventBatch]:
dynamodb = get_boto3_client("dynamodb")
table_name = f"dialog-event-batches-{stage}"
args: Dict[str, str] = {}
while True:
result = dynamodb.query(
TableName=table_name,
IndexName="by_created_time",
KeyConditionExpression="phone_number=:phone_number",
ExpressionAttributeValues={":phone_number": {"S": phone_number}},
**args,
)
for item in result["Items"]:
yield batch_from_dict(dynamodb_utils.deserialize(item))
if not result.get("LastEvaluatedKey"):
break
args["ExclusiveStartKey"] = result["LastEvaluatedKey"]
def handler(event: dict, context: dict) -> dict:
verify_deploy_stage()
kinesis = get_boto3_client("kinesis")
stage = os.environ["STAGE"]
idempotency_checker = IdempotencyChecker()
form = extract_form(event)
if not is_signature_valid(event, form, stage):
logging.warning("signature validation failed")
return {"statusCode": 403}
idempotency_key = event["headers"]["I-Twilio-Idempotency-Token"]
if idempotency_checker.already_processed(idempotency_key,
IDEMPOTENCY_REALM):
logging.info(
f"Already processed webhook with idempotency key {idempotency_key}. Skipping."
)
return {"statusCode": 200}
if "MessageStatus" in form:
logging.info(
f"Outbound message to {form['To']}: Recording STATUS_UPDATE in message log"
)
kinesis.put_record(
Data=json.dumps({
"type": "STATUS_UPDATE",
"received_at": datetime.now(UTC).isoformat(),
"payload": form,
}),
PartitionKey=form["To"],
StreamName=f"message-log-{stage}",
)
else:
logging.info(f"Inbound message from {form['From']}: '{form['Body']}'")
CommandPublisher().publish_process_sms_command(form["From"],
form["Body"], form)
idempotency_checker.record_as_processed(idempotency_key, IDEMPOTENCY_REALM,
IDEMPOTENCY_EXPIRATION_MINUTES)
return {
"statusCode": 200,
"headers": {
"content-type": "application/xml"
},
"body": str(MessagingResponse()),
}
def read_messages(phone: str, localstack: str) -> None:
sqs = get_boto3_client("sqs")
queue_url = f"{localstack}{QUEUE_URL_PATH}"
while True:
res = sqs.receive_message(QueueUrl=queue_url)
messages = res.get("Messages")
if messages:
for message in messages:
body = json.loads(message["Body"])
phone_number = body["phone_number"]
if phone_number == phone:
print(GREEN)
print(f"Inbound to {phone_number}:")
for m in body["messages"]:
print(m["body"])
if m["media_url"]:
print(m["media_url"])
print(END_COLOR)
sqs.delete_message(QueueUrl=queue_url,
ReceiptHandle=message["ReceiptHandle"])
def __init__(self, table_name_suffix: str = None, **kwargs: Any) -> None:
self.dynamodb = get_boto3_client("dynamodb", **kwargs)
if table_name_suffix is None:
table_name_suffix = os.getenv("DIALOG_TABLE_NAME_SUFFIX", "")
self.table_name_suffix = table_name_suffix
def __init__(self, **kwargs: Any) -> None:
self.dynamodb = get_boto3_client("dynamodb", **kwargs)
self.stage = os.environ.get("STAGE")