def handle_replay_sqs_failures(args: Any) -> None: sqs = get_boto3_resource("sqs") kinesis = get_boto3_client("kinesis") queue_name = f"{args.sqs_queue}-failures-{args.stage}" stream_name = f"{args.kinesis_stream}-{args.stage}" queue = sqs.get_queue_by_name(QueueName=queue_name) while True: print(f"Getting messages from {queue_name}...") messages = queue.receive_messages(WaitTimeSeconds=1) if not messages: print(f"Reached end of {queue_name}") return for message in messages: print(f"Message ID: {message.message_id}") print(json.dumps(json.loads(message.body), indent=4, sort_keys=True)) if args.print_only: continue response = input( f"Re-publish message {message.message_id} to {stream_name}? (yes/no)\n" ).lower() if response in ["y", "yes"]: partition_key = input(f"What partition in {args.kinesis_stream}?\n").lower() records = [{"Data": message.body, "PartitionKey": partition_key}] print( f"Re-publishing message {message.message_id} to {stream_name} at partition {partition_key}" ) response = kinesis.put_records(StreamName=stream_name, Records=records) print(f"Deleting message {message.message_id}\n") message.delete() else: print(f"Skipping message {message.message_id}\n")
def handler(event: dict, context: dict) -> dict: verify_deploy_stage() kinesis = get_boto3_client("kinesis") stage = os.environ["STAGE"] idempotency_checker = IdempotencyChecker() inbound_commands = [ _make_inbound_command(record) for record in event["Records"] ] for command in inbound_commands: if command.command_type is InboundCommandType.INBOUND_SMS: if not idempotency_checker.already_processed( command.sequence_number, IDEMPOTENCY_REALM): twilio_webhook = command.payload["twilio_webhook"] kinesis.put_record( Data=json.dumps({ "type": "INBOUND_SMS", "payload": twilio_webhook }), PartitionKey=command.payload["From"], StreamName=f"message-log-{stage}", ) idempotency_checker.record_as_processed( command.sequence_number, IDEMPOTENCY_REALM, IDEMPOTENCY_EXPIRATION_MINUTES, ) return {"statusCode": 200}
def _publish_commands(self, commands: List[Tuple[str, Dict[str, Any]]]) -> None: kinesis = get_boto3_client("kinesis") records = [{ "Data": json.dumps(data), "PartitionKey": phone_number } for phone_number, data in commands] response = kinesis.put_records( StreamName=f"command-stream-{self.stage}", Records=records) if response.get("FailedRecordCount"): rollbar.report_exc_info(extra_data=response)
def get_all_users(args: Any) -> None: dynamodb = get_boto3_client("dynamodb") table_name = f"dialog-state-{args.stage}" args = {} while True: result = dynamodb.scan(TableName=table_name, **args) for item in result["Items"]: print(item["phone_number"]["S"]) if not result.get("LastEvaluatedKey"): break args["ExclusiveStartKey"] = result["LastEvaluatedKey"]
def handle_show_stream_record(args: Any) -> None: kinesis = get_boto3_client("kinesis") stream_name = f"{args.kinesis_stream}-{args.stage}" shard_iterator = kinesis.get_shard_iterator( StreamName=stream_name, ShardId=args.shard_id, ShardIteratorType="AT_SEQUENCE_NUMBER", StartingSequenceNumber=args.seq, ) response = kinesis.get_records(ShardIterator=shard_iterator["ShardIterator"], Limit=1) for record in response["Records"]: print(json.loads(record["Data"]))
def _publish_event_batches_to_kinesis(event_batches: List[DialogEventBatch]) -> None: kinesis = get_boto3_client("kinesis") stage = os.environ.get("STAGE") stream_name = f"dialog-event-batches-{stage}" records = [ { "PartitionKey": event_batch.phone_number, "Data": event_batch.json(), } for event_batch in event_batches ] response = kinesis.put_records(StreamName=stream_name, Records=records) if response.get("FailedRecordCount"): rollbar.report_exc_info(extra_data=response)
def publish_outbound_sms(payload: OutboundPayload) -> dict: kinesis = get_boto3_client("kinesis") stage = os.environ.get("STAGE") records = [ { "Data": json.dumps( { "type": "OUTBOUND_SMS", "payload": payload.dict(), } ), "PartitionKey": payload.To, } ] return cast(dict, kinesis.put_records(Records=records, StreamName=f"message-log-{stage}"))
def _get_dialog_events(phone_number: str, stage: str) -> Iterator[DialogEventBatch]: dynamodb = get_boto3_client("dynamodb") table_name = f"dialog-event-batches-{stage}" args: Dict[str, str] = {} while True: result = dynamodb.query( TableName=table_name, IndexName="by_created_time", KeyConditionExpression="phone_number=:phone_number", ExpressionAttributeValues={":phone_number": {"S": phone_number}}, **args, ) for item in result["Items"]: yield batch_from_dict(dynamodb_utils.deserialize(item)) if not result.get("LastEvaluatedKey"): break args["ExclusiveStartKey"] = result["LastEvaluatedKey"]
def handler(event: dict, context: dict) -> dict: verify_deploy_stage() kinesis = get_boto3_client("kinesis") stage = os.environ["STAGE"] idempotency_checker = IdempotencyChecker() form = extract_form(event) if not is_signature_valid(event, form, stage): logging.warning("signature validation failed") return {"statusCode": 403} idempotency_key = event["headers"]["I-Twilio-Idempotency-Token"] if idempotency_checker.already_processed(idempotency_key, IDEMPOTENCY_REALM): logging.info( f"Already processed webhook with idempotency key {idempotency_key}. Skipping." ) return {"statusCode": 200} if "MessageStatus" in form: logging.info( f"Outbound message to {form['To']}: Recording STATUS_UPDATE in message log" ) kinesis.put_record( Data=json.dumps({ "type": "STATUS_UPDATE", "received_at": datetime.now(UTC).isoformat(), "payload": form, }), PartitionKey=form["To"], StreamName=f"message-log-{stage}", ) else: logging.info(f"Inbound message from {form['From']}: '{form['Body']}'") CommandPublisher().publish_process_sms_command(form["From"], form["Body"], form) idempotency_checker.record_as_processed(idempotency_key, IDEMPOTENCY_REALM, IDEMPOTENCY_EXPIRATION_MINUTES) return { "statusCode": 200, "headers": { "content-type": "application/xml" }, "body": str(MessagingResponse()), }
def read_messages(phone: str, localstack: str) -> None: sqs = get_boto3_client("sqs") queue_url = f"{localstack}{QUEUE_URL_PATH}" while True: res = sqs.receive_message(QueueUrl=queue_url) messages = res.get("Messages") if messages: for message in messages: body = json.loads(message["Body"]) phone_number = body["phone_number"] if phone_number == phone: print(GREEN) print(f"Inbound to {phone_number}:") for m in body["messages"]: print(m["body"]) if m["media_url"]: print(m["media_url"]) print(END_COLOR) sqs.delete_message(QueueUrl=queue_url, ReceiptHandle=message["ReceiptHandle"])
def __init__(self, table_name_suffix: str = None, **kwargs: Any) -> None: self.dynamodb = get_boto3_client("dynamodb", **kwargs) if table_name_suffix is None: table_name_suffix = os.getenv("DIALOG_TABLE_NAME_SUFFIX", "") self.table_name_suffix = table_name_suffix
def __init__(self, **kwargs: Any) -> None: self.dynamodb = get_boto3_client("dynamodb", **kwargs) self.stage = os.environ.get("STAGE")