async def consume_forever(consumer: AIOKafkaConsumer):
""" Infinite loop reading the messages
sent from the flink cluster back to the application"""
# Iterate through the messages and change the
# future of the dict to be this result
logger.info('Consumer starting to consumer messages')
async for msg in consumer:
# if the message is for our worker, get it
if msg.key.decode('utf-8') == WORKER_ID:
# logger.info(f'Received message! {msg.value}')
resp = ResponseMessage()
resp.ParseFromString(msg.value)
# set the result of the future in the dict
if resp.request_id in messages:
# logger.info(f'Setting future for message {resp.request_id}')
if messages[resp.request_id].done():
logger.warning(
f'Future was already done while setting it {resp.request_id}'
)
else:
messages[resp.request_id].set_result(resp.result)
else:
logger.error(
f'Received response for an unknown message with request id: {resp.request_id}'
)
def remove_order(context, msg):
state = context.state('order').unpack(Order)
response = ResponseMessage()
if not state:
response.result = json.dumps({
'result': 'failure',
'message': 'Order does not exist.'
})
else:
#logger.info(f"Deleting the order with id: {msg.remove_order.id}")
item_to_count = {}
items = state.items
for i in range(len(items)):
id = items[i].item_id
if id not in item_to_count.keys():
item_to_count[id] = 1
else:
item_to_count[id] = item_to_count[id] + 1
for id, cnt in item_to_count.items():
add_stock_request = OrderAddItemStockRequest()
add_stock_request.request_info.worker_id = msg.request_info.worker_id
add_stock_request.request_info.request_id = msg.request_info.request_id
add_stock_request.id = id
add_stock_request.amount = cnt
# logger.info('Sending request to add stock back.')
context.pack_and_send("stock/stock", str(id), add_stock_request)
# logger.info('Deleting order.')
del context['order']
response.result = json.dumps({'result': 'success'})
return response
def create_order_with_id(context, msg):
state = Order()
state.id = msg.id
state.user_id = msg.user_id
state.paid = False
state.total_cost = 0
context.state('order').pack(state)
#logger.debug(f'Created new order with id {msg.id}')
response = ResponseMessage()
response.result = json.dumps({'result': 'success', 'order_id': state.id})
return response
def remove_item(context, msg):
orderId = msg.remove_item.id
state = context.state('order').unpack(Order)
response = ResponseMessage()
if not state:
response.result = json.dumps({
'result': 'failure',
'message': 'Order does not exist.'
})
else:
items = state.items
item_to_delete = msg.remove_item.itemId
item_index = -1
for i in range(len(items)):
#logger.info(f"{items[i].item_id}")
if items[i].item_id == item_to_delete:
item_index = i
if item_index != -1:
state.total_cost -= items[item_index].price
del state.items[item_index]
#logger.info(f"Removing item {item_to_delete} from order {orderId}")
response.result = json.dumps({'result': 'success'})
add_stock_request = StockRequest()
add_stock_request.request_info.worker_id = msg.request_info.worker_id
add_stock_request.request_info.request_id = msg.request_info.request_id
add_stock_request.add_stock.id = item_to_delete
add_stock_request.add_stock.amount = 1
context.pack_and_send("stock/stock", str(item_to_delete),
add_stock_request)
else:
#logger.warning(f"Order {orderId} does not contain item with id {item_to_delete}")
response.result = json.dumps({
'result':
'failure',
'message':
'Order does not contain requested item.'
})
context.state('order').pack(state)
return response
def order_payment_confirm(context, msg):
response = ResponseMessage()
if msg.actually_paid:
state = context.state('order').unpack(Order)
state.paid = True
context.state('order').pack(state)
#logger.debug("Checkout succeeded.")
response.result = json.dumps({
'result': 'success',
'message': 'Checkout succeeded.'
})
else:
#logger.debug("Payment cancelling failed.")
response.result = json.dumps({
'result': 'failure',
'message': 'Payment cancelling failed.'
})
return response
def order_checkout(context, msg):
state = context.state('order').unpack(Order)
if not state:
response = ResponseMessage()
response.result = json.dumps({
'result': 'failure',
'message': 'Order does not exist.'
})
return response
if state.paid:
response = ResponseMessage()
response.result = json.dumps({
'result': 'failure',
'message': 'Order already paid.'
})
return response
request = Order()
request.id = msg.order_checkout.id
request.request_info.worker_id = msg.request_info.worker_id
request.request_info.request_id = msg.request_info.request_id
request.user_id = state.user_id
# to assign a repeated field we need to call extend
request.items.extend(state.items)
request.total_cost = state.total_cost
request.paid = state.paid
request.intent = Order.Intent.PAY
#logger.debug(f'Sending order check to payments {request}')
# send to payment service
context.pack_and_send("payments/pay", str(request.id), request)
def order_add_item_reply(context, msg):
state = context.state('order').unpack(Order)
response = ResponseMessage()
if msg.result == 'success':
#logger.info("Successfully added item to order.")
new_item = Item()
new_item.item_id = msg.item_id
new_item.price = msg.price
state.items.append(new_item)
state.total_cost += msg.price
#logger.info(f"{state}")
context.state('order').pack(state)
response.result = json.dumps({'result': 'success'})
else:
#logger.debug("No items left in stock.")
response.result = json.dumps({
'result': 'failure',
'message': 'No items left in stock.'
})
context.state('order').pack(state)
return response
def find_order(context, msg):
state = context.state('order').unpack(Order)
response = ResponseMessage()
if not state:
response.result = json.dumps({
'result': 'failure',
'message': 'Order does not exist.'
})
else:
# Have to assign all like this so they're not casted to string
response.result = json.dumps({
'id': state.id,
'user_id': state.user_id,
'items': [i.item_id for i in state.items],
'total_cost': state.total_cost,
'paid': state.paid,
'intent': state.intent
})
context.state('order').pack(state)
return response
def add_item(context, msg):
state = context.state('order').unpack(Order)
if not state:
response = ResponseMessage()
response.result = json.dumps({
'result': 'failure',
'message': 'Order does not exist.'
})
return response
else:
# call stock service to reduce the stock
subtract_stock_request = StockRequest()
subtract_stock_request.request_info.worker_id = msg.request_info.worker_id
subtract_stock_request.request_info.request_id = msg.request_info.request_id
subtract_stock_request.subtract_stock.id = msg.add_item.itemId
subtract_stock_request.subtract_stock.amount = 1
subtract_stock_request.internal = True
subtract_stock_request.order_id = state.id
context.state('order').pack(state)
context.pack_and_send("stock/stock", str(msg.add_item.itemId),
subtract_stock_request)
def manage_stock(context, request: typing.Union[StockRequest, CreateItemRequest, OrderAddItemStockRequest]):
# Get the current state.
item_state: ItemData = context.state('item').unpack(ItemData)
if isinstance(request, CreateItemRequest):
item_state = ItemData()
item_state.id = request.id
item_state.price = request.price
item_state.stock = 0
context.state('item').pack(item_state)
#logger.debug(f'Created new item with id {request.id}')
response = ResponseMessage()
response.result = json.dumps({'item_id': item_state.id})
elif isinstance(request, OrderAddItemStockRequest):
response = None
if item_state is None:
pass
else:
item_state.stock += request.amount
context.state('item').pack(item_state)
elif isinstance(request, StockRequest):
# If the item state is None we return an error
if item_state is None:
# Item does not exist yet. Return error.
if not request.internal:
response = ResponseMessage()
response.result = json.dumps({'result': 'not_found'})
else:
response = StockResponse()
response.item_id = request.subtract_stock.id
response.result = 'failure'
else:
# check which field we have
msg_type = request.WhichOneof('message')
#logger.debug(f'Got message of type {msg_type}')
if msg_type == "find_item":
response = ResponseMessage()
response.result = json.dumps(
{'id:': item_state.id, 'price': item_state.price, 'stock': item_state.stock})
context.state('item').pack(item_state)
elif msg_type == "subtract_stock":
new_amount = item_state.stock - request.subtract_stock.amount
if not request.internal:
response = ResponseMessage()
else:
response = StockResponse()
if new_amount >= 0:
item_state.stock -= request.subtract_stock.amount
context.state('item').pack(item_state)
if not request.internal:
response.result = json.dumps({'result': 'success', 'item_id': item_state.id})
else:
# Include the item id and price
response.price = item_state.price
response.item_id = item_state.id
response.result = 'success'
else:
if not request.internal:
response.result = json.dumps({'result': 'stock too low', 'item_id': item_state.id})
else:
response.price = item_state.price
response.item_id = item_state.id
response.result = 'failure'
elif msg_type == "add_stock":
item_state.stock += request.add_stock.amount
context.state('item').pack(item_state)
# send the response.
response = ResponseMessage()
response.result = json.dumps({'result': 'success', 'item_id': item_state.id})
if response:
# Use the same request id in the message body
# and use the request worker_id as key of the message
if not request.internal:
response.request_id = request.request_info.request_id
# create the egress message and send it to the
# users/out egress
egress_message = kafka_egress_record(
topic=STOCK_EVENTS_TOPIC,
key=request.request_info.worker_id,
value=response
)
context.pack_and_send_egress("stock/out", egress_message)
else:
response.request_info.request_id = request.request_info.request_id
response.request_info.worker_id = request.request_info.worker_id
context.pack_and_send("orders/order", str(request.order_id), response)
def payments_pay(context,
request: typing.Union[PaymentRequest, UserPayRequest, Order,
OrdersPayFind, UserPayResponse,
OrderPaymentCancelReply]):
# Incoming from Orders
if isinstance(request, Order):
# If intent == PAY, request is initiated by Orders
if request.intent == Order.Intent.PAY:
user_pay_request = set_worker_and_request_ids(
request, UserPayRequest())
user_pay_request.order_id = request.id
user_pay_request.amount = request.total_cost
context.pack_and_send("users/user", str(request.user_id),
user_pay_request)
# If intent == CANCEL or STATUS, this is a reply message to an earlier request
elif request.intent == Order.Intent.CANCEL:
if not request.paid:
# Payment cannot be cancelled cause it is not paid
response = ResponseMessage()
response.request_id = request.request_info.request_id
response.result = json.dumps({'result': 'failure'})
send_response(context, response,
request.request_info.worker_id)
# Otherwise send request to user to subtract the amount
elif request.paid:
user_pay_request = set_worker_and_request_ids(
request, UserCancelPayRequest())
user_pay_request.order_id = request.order_id
user_pay_request.amount = request.total_cost
context.pack_and_send("users/user", request.user_id,
user_pay_request)
elif request.intent == Order.Intent.STATUS:
response = ResponseMessage()
response.request_id = request.request_info.request_id
response.result = json.dumps({
'paid': True
}) if request.paid else json.dumps({'paid': False})
send_response(context, response, request.request_info.worker_id)
# Incoming from Flask
elif isinstance(request, PaymentRequest):
if request.request_type == PaymentRequest.RequestType.CANCEL:
order_payment_cancel_request = set_worker_and_request_ids(
request, OrderPaymentCancel())
order_payment_cancel_request.order_id = request.order_id
context.pack_and_send("orders/order", request.order_id,
order_payment_cancel_request)
elif request.request_type == PaymentRequest.RequestType.STATUS:
orders_pay_find_request = set_worker_and_request_ids(
request, OrdersPayFind())
orders_pay_find_request.order_id = request.order_id
context.pack_and_send("orders/order", request.order_id,
orders_pay_find_request)
# Reply from Users
elif isinstance(request, UserPayResponse):
payment_status = set_worker_and_request_ids(request, PaymentStatus())
payment_status.order_id = request.order_id
payment_status.actually_paid = request.success
context.pack_and_send("orders/order", request.order_id, payment_status)
# Reply from Users
elif isinstance(request, OrderPaymentCancelReply):
response = ResponseMessage()
response.request_id = request.request_info.request_id
response.result = json.dumps({
'result': 'success'
}) if request.success else json.dumps({'result': 'failure'})
send_response(context, response, request.request_info.worker_id)
def operate_user(context,
request: typing.Union[UserPayRequest, UserCancelPayRequest,
UserRequest, CreateUserRequest]):
""" Does all the operations with a single user
Has the state of a user in a UserData() object that includes
its id and credit under the name 'user' """
# Get the current state for a given user
# could have to handle the state not existing for this user
state: UserData = context.state('user').unpack(UserData)
response = None
# ----------------------------------------
# Messages from the payment endpoint
# ----------------------------------------
if isinstance(request, UserPayRequest):
#logger.debug('Received request to decrement user credit')
# calculate if the credit is enough to pay for the product
# get the credit
response = UserPayResponse()
response.order_id = request.order_id
# copy the information of the request_info
response = copy_request_info(request, response)
# if the user exists then do the checks
if state:
# see whether we should return success or failure
if state.credit - request.amount < 0:
response.success = False
else:
state.credit -= request.amount
response.success = True
else:
response.success = False
# pack the state
context.state('user').pack(state)
# respond to the payment service
context.pack_and_reply(response)
return
elif isinstance(request, UserCancelPayRequest):
#logger.debug('Received request to cancel a payment')
# add the amount specified to the user credit
response = UserPayResponse()
response.order_id = request.order_id
# copy the information
response = copy_request_info(request, response)
if state:
state.credit += request.amount
# reply
response.success = True
else:
response.success = False
# pack the state
context.state('user').pack(state)
# reply to the sender function
context.pack_and_reply(response)
return
# -------------------------------------
# Interaction with the user endpoint
# -------------------------------------
elif isinstance(request, CreateUserRequest):
# we are given the uuid in the message so that's already done
state = UserData()
state.id = request.id
state.credit = 0
#logger.debug(f'Created new user with id {request.id}')
context.state('user').pack(state)
response = ResponseMessage()
response.result = json.dumps({'user_id': state.id})
elif isinstance(request, UserRequest):
# check which field we have
msg_type = request.WhichOneof('message')
#logger.debug(f'Got message of type {msg_type}')
# If the state is None we return an error
if not state:
response = ResponseMessage()
response.result = json.dumps(
{'result': 'failure: user does not exist'})
else:
# If the state exists we then operate with it
if msg_type == 'find_user':
response = ResponseMessage()
response.result = json.dumps({
'user_id': state.id,
'credit': state.credit
})
# pack the state
context.state('user').pack(state)
elif msg_type == 'remove_user':
del context['user']
response = ResponseMessage()
response.result = json.dumps({'result': 'success'})
elif msg_type == 'add_credit':
# Update the credit and save state
state.credit += request.add_credit.amount
context.state('user').pack(state)
# send the response
response = ResponseMessage()
response.result = json.dumps({'result': 'success'})
elif msg_type == 'subtract_credit':
# try to subtract the amount from the user credit
new_amount = state.credit - request.subtract_credit.amount
response = ResponseMessage()
if new_amount >= 0:
state.credit -= request.subtract_credit.amount
context.state('user').pack(state)
response.result = json.dumps({'result': 'success'})
else:
response.result = json.dumps({'result': 'failure'})
else:
logger.error('Received unknown message type!')
# respond if needed
if response:
# Use the same request id in the message body
# and use the request worker_id as key of the message
response.request_id = request.request_info.request_id
# create the egress message and send it to the
# users/out egress
egress_message = kafka_egress_record(
topic=USER_EVENTS_TOPIC,
key=request.request_info.worker_id,
value=response)
context.pack_and_send_egress("users/out", egress_message)