async def handle_transfer(context, message: Transfer, request_id: str) -> None:
# Send messages
subtract_credit = SubtractCredit(amount=message.amount)
context.pack_and_send_atomic_invocation("ycsb-example/account_function",
message.outgoing_id,
subtract_credit)
add_credit = AddCredit(amount=message.amount)
context.pack_and_send_atomic_invocation("ycsb-example/account_function",
message.incoming_id, add_credit)
# Send on success
response = Response(request_id=request_id, status_code=200)
egress_message = kafka_egress_record(topic="responses",
key=request_id,
value=response)
context.pack_and_send_egress_on_success("ycsb-example/kafka-egress",
egress_message)
# Send on failure
response = Response(request_id=request_id, status_code=422)
egress_message = kafka_egress_record(topic="responses",
key=request_id,
value=response)
context.pack_and_send_egress_on_failure("ycsb-example/kafka-egress",
egress_message)
# Send on retryable (e.g. deadlock)
response = Response(request_id=request_id, status_code=401)
egress_message = kafka_egress_record(topic="responses",
key=request_id,
value=response)
context.pack_and_send_egress_on_retryable("ycsb-example/kafka-egress",
egress_message)
def transfer_function(context, request: Transfer):
# Send messages
delete = Delete()
context.pack_and_send_atomic_invocation("ycsb-example/account_function",
request.outgoing_id, delete)
add_credit = AddCredit(amount=request.amount)
context.pack_and_send_atomic_invocation("ycsb-example/account_function",
request.incoming_id, add_credit)
# Send on success
response = Response(request_id=context.address.identity, status_code=200)
egress_message = kafka_egress_record(topic="responses",
key=context.address.identity,
value=response)
context.pack_and_send_egress("ycsb-example/kafka-egress",
egress_message,
success=True)
# Send on failure
response = Response(request_id=context.address.identity, status_code=422)
egress_message = kafka_egress_record(topic="responses",
key=context.address.identity,
value=response)
context.pack_and_send_egress("ycsb-example/kafka-egress",
egress_message,
success=False)
def egress(context, message):
# send a message to an external system via an egress. Egresses needs to be defined in a module.yaml
# and can be referenced by type.
# The following two lines prepare a message to send to the pre-built Kafka egress.
key = context.address.identity # use the identity part of our own address as the target Kafka key.
record = kafka_egress_record(topic="events", key=key, value=Event())
context.pack_and_send_egress("walkthrough/events-egress", record)
def overview(context, state: OrderStateWithPrevious):
if state.status == "UNASSIGNED":
no_unassigned = context.state('no_unassigned').unpack(NoState)
if not no_unassigned:
no_unassigned = NoState()
no_unassigned.counter = 1
else:
no_unassigned.counter += 1
context.state('no_unassigned').pack(no_unassigned)
elif state.status == "ASSIGNED":
no_unassigned = context.state('no_unassigned').unpack(NoState)
if no_unassigned and state.previous:
no_unassigned.counter -= 1
context.state('no_unassigned').pack(no_unassigned)
no_assigned = context.state('no_assigned').unpack(NoState)
if not no_assigned:
no_assigned = NoState()
no_assigned.counter = 1
else:
no_assigned.counter += 1
context.state('no_assigned').pack(no_assigned)
elif state.status == "IN_PROGRESS":
no_assigned = context.state('no_assigned').unpack(NoState)
if no_assigned and state.previous:
no_assigned.counter -= 1
context.state('no_assigned').pack(no_assigned)
no_in_progress = context.state('no_in_progress').unpack(NoState)
if not no_in_progress:
no_in_progress = NoState()
no_in_progress.counter = 1
else:
no_in_progress.counter += 1
context.state('no_in_progress').pack(no_in_progress)
elif state.status == "DELIVERED":
no_in_progress = context.state('no_in_progress').unpack(NoState)
if no_in_progress and state.previous:
no_in_progress.counter -= 1
context.state('no_in_progress').pack(no_in_progress)
no_delivered = context.state('no_delivered').unpack(NoState)
if not no_delivered:
no_delivered = NoState()
no_delivered.counter = 1
else:
no_delivered.counter += 1
context.state('no_delivered').pack(no_delivered)
overview = compute_overview(context)
egress_message = kafka_egress_record(topic="overviews",
key="overview",
value=overview)
context.pack_and_send_egress("lieferbot/status", egress_message)
def send_response(context, response_message, worker_id):
#logger.debug(worker_id)
egress_message = kafka_egress_record(topic=PAYMENT_EVENTS_TOPIC,
key=worker_id,
value=response_message)
#logger.info(f'Sending message {egress_message}')
context.pack_and_send_egress("payments/out", egress_message)
def forward_to_egress(context, invoke_result: InvokeResult):
"""
Simply forwards the results to the Kafka egress.
"""
egress_message = kafka_egress_record(topic="invoke-results",
key=invoke_result.id,
value=invoke_result)
context.pack_and_send_egress(
"org.apache.flink.statefun.e2e.remote/invoke-results", egress_message)
def score(context, msg: ServerMetricHistory):
print(msg)
if not repair(msg):
return
request = RepairServer()
request.server_id = context.address.identity
message = kafka_egress_record(topic="repair", key=request.server_id, value=request)
print("fixing " + request.server_id)
context.pack_and_send_egress("io/repairs", message)
def monitor(context, order_update: OrderUpdate):
state = context.state('order_state').unpack(OrderState)
state_with_previous = OrderStateWithPrevious()
state_with_previous.status = order_update.status
if not state:
state = OrderState()
state_with_previous.previous = False
else:
state_with_previous.previous = True
state.status = order_update.status
context.state('order_state').pack(state)
if state.status == "UNASSIGNED":
time_unassigned = context.state('time_unassigned').unpack(Time)
if not time_unassigned:
time_unassigned = Time()
time_unassigned.time = order_update.time
context.state('time_unassigned').pack(time_unassigned)
elif state.status == "ASSIGNED":
time_assigned = context.state('time_assigned').unpack(Time)
if not time_assigned:
time_assigned = Time()
time_assigned.time = order_update.time
context.state('time_assigned').pack(time_assigned)
elif state.status == "IN_PROGRESS":
time_in_progress = context.state('time_in_progress').unpack(Time)
if not time_in_progress:
time_in_progress = Time()
time_in_progress.time = order_update.time
context.state('time_in_progress').pack(time_in_progress)
elif state.status == "DELIVERED":
time_delivered = context.state('time_delivered').unpack(Time)
if not time_delivered:
time_delivered = Time()
time_delivered.time = order_update.time
context.state('time_delivered').pack(time_delivered)
report = compute_report(context, order_update)
egress_message = kafka_egress_record(topic="reports",
key=order_update.id,
value=report)
context.pack_and_send_egress("lieferbot/status", egress_message)
context.pack_and_send("lieferbot/overview", "overview",
state_with_previous)
context.pack_and_send("lieferbot/timeout_counter", order_update.id,
order_update)
context.pack_and_send("lieferbot/timeout_check", order_update.id, state)
def fun(context, message):
# state access
seen = context.state('seen').unpack(SeenCount)
seen.seen += 1
context.state('seen').pack(seen)
# regular state access
seenAny = context['seen']
seenAny.Unpack(seen)
# sending and replying
context.pack_and_reply(seen)
any = Any()
any.type_url = 'type.googleapis.com/k8s.demo.SeenCount'
context.send("bar.baz/foo", "12345", any)
# delayed messages
context.send_after(timedelta(hours=1), "night/owl", "1", any)
# egresses
context.send_egress("foo.bar.baz/my-egress", any)
context.pack_and_send_egress("foo.bar.baz/my-egress", seen)
# kafka egress
context.pack_and_send_egress("sdk/kafka",
kafka_egress_record(topic="hello", key=u"hello world", value=seen))
context.pack_and_send_egress("sdk/kafka",
kafka_egress_record(topic="hello", value=seen))
# AWS Kinesis generic egress
context.pack_and_send_egress("sdk/kinesis",
kinesis_egress_record(
stream="hello",
partition_key=u"hello world",
value=seen,
explicit_hash_key=u"1234"))
context.pack_and_send_egress("sdk/kinesis",
kinesis_egress_record(
stream="hello",
partition_key=u"hello world",
value=seen))
def greet(context, message: LoginEvent):
state = context.state('seen_count').unpack(SeenCount)
if not state:
state = SeenCount()
state.seen = 1
else:
state.seen += 1
context.state('seen_count').pack(state)
egress_message = kafka_egress_record(topic="seen", key=message.user_name, value=state)
context.pack_and_send_egress("k8s-demo/greets-egress", egress_message)
async def send_response(context,
request_id: str,
status_code: int,
message=None) -> None:
response = Response(request_id=request_id, status_code=status_code)
if message:
out = Any()
out.Pack(message)
response.message.CopyFrom(out)
egress_message = kafka_egress_record(topic="responses",
key=request_id,
value=response)
context.pack_and_send_egress("ycsb-example/kafka-egress", egress_message)
def operate_order(context, msg: typing.Union[CreateOrder, OrderRequest,
OrdersPayFind, OrderPaymentCancel,
PaymentStatus, StockResponse]):
""" Does all the operations with a single order """
response = None
if isinstance(msg, CreateOrder):
response = create_order_with_id(context, msg)
elif isinstance(msg, OrderRequest):
msg_type = msg.WhichOneof('message')
# logger.debug(f'Got message of type {msg_type}')
if msg_type == 'remove_order':
response = remove_order(context, msg)
elif msg_type == 'find_order':
response = find_order(context, msg)
elif msg_type == 'add_item':
response = add_item(context, msg)
elif msg_type == 'remove_item':
response = remove_item(context, msg)
elif msg_type == 'order_checkout':
response = order_checkout(context, msg)
elif isinstance(msg, OrdersPayFind):
order_payment_find(context, msg)
elif isinstance(msg, OrderPaymentCancel):
order_payment_cancel(context, msg)
elif isinstance(msg, PaymentStatus):
response = order_payment_confirm(context, msg)
elif isinstance(msg, StockResponse):
response = order_add_item_reply(context, msg)
else:
logger.error('Received unknown message type!')
if response:
response.request_id = msg.request_info.request_id
egress_message = kafka_egress_record(topic=ORDER_EVENTS_TOPIC,
key=msg.request_info.worker_id,
value=response)
context.pack_and_send_egress("orders/out", egress_message)
def greet(context, greet_request: GreetRequest):
state = context.state('seen_count').unpack(SeenCount)
if not state:
state = SeenCount()
state.seen = 1
else:
state.seen += 1
context.state('seen_count').pack(state)
response = compute_greeting(greet_request.name, state.seen)
egress_message = kafka_egress_record(topic="greetings",
key=greet_request.name,
value=response)
context.pack_and_send_egress("example/greets", egress_message)
def check_request(context, transfer_request: TransferRequest):
state = context.state('amount').unpack(AmountTransfered)
if not state:
state = AmountTransfered()
state.amount = transfer_request.amount
else:
state.amount += transfer_request.amount
context.state('amount').pack(state)
response = send_acknowledgement(transfer_request.number,
transfer_request.amount, state.amount)
egress_message = kafka_egress_record(topic="transfer_response",
key=transfer_request.number,
value=response)
context.pack_and_send_egress("demo/transfer/acknowledgement",
egress_message)
async def account_function(context, request: Wrapper):
# Get state
state = context.state('state').unpack(State)
# messages from outside
request_id = request.request_id
message = request.message
if message.Is(Transfer.DESCRIPTOR):
transfer = Transfer()
message.Unpack(transfer)
read = Read()
read.id = transfer.outgoing_id
wrapped = Wrapper()
wrapped.request_id = request_id
outgoing_message = Any()
outgoing_message.Pack(read)
wrapped.message.CopyFrom(outgoing_message)
context.pack_and_send("ycsb-example/account_function",
transfer.outgoing_id, wrapped)
elif message.Is(Read.DESCRIPTOR):
read = Read()
message.Unpack(read)
update = Update()
wrapped = Wrapper()
wrapped.request_id = request_id
outgoing_message = Any()
outgoing_message.Pack(update)
wrapped.message.CopyFrom(outgoing_message)
context.pack_and_reply(wrapped)
elif message.Is(Update.DESCRIPTOR):
update = Update()
message.Unpack(update)
response = Response(request_id=request_id, status_code=200)
egress_message = kafka_egress_record(topic="responses",
key=request_id,
value=response)
context.pack_and_send_egress("ycsb-example/kafka-egress",
egress_message)
else:
print("Test test test")
def greet(context, greet_request: GreetRequest):
state = context.state('seen_count').unpack(SeenCount)
if not state:
state = SeenCount()
state.seen = 1
state.sum = 0
else:
state.seen += 1
state.sum += greet_request.val
context.state('seen_count').pack(state)
response = compute_greeting(greet_request.name, state.seen)
response.val = greet_request.val
egress_message = kafka_egress_record(topic="greetings",
key="vals".encode('utf-8'),
value=state)
context.pack_and_send_egress("example/greets", egress_message)
def timeout(context, request: typing.Union[OrderState, OrderUpdate]):
if isinstance(request, OrderState):
state = context.state('order_state').unpack(OrderState)
if not state:
state = OrderState()
state.status = request.status
context.state('order_state').pack(state)
elif isinstance(request, OrderUpdate):
state = context.state('order_state').unpack(OrderState)
if (state.status == request.status):
report = TimeoutReport()
report.order.status = state.status
report.orderId = request.id
egress_message = kafka_egress_record(topic="timeouts",
key=request.id,
value=report)
context.pack_and_send_egress("lieferbot/status", egress_message)
def manage_stock(context, request: typing.Union[StockRequest, CreateItemRequest, OrderAddItemStockRequest]):
# Get the current state.
item_state: ItemData = context.state('item').unpack(ItemData)
if isinstance(request, CreateItemRequest):
item_state = ItemData()
item_state.id = request.id
item_state.price = request.price
item_state.stock = 0
context.state('item').pack(item_state)
#logger.debug(f'Created new item with id {request.id}')
response = ResponseMessage()
response.result = json.dumps({'item_id': item_state.id})
elif isinstance(request, OrderAddItemStockRequest):
response = None
if item_state is None:
pass
else:
item_state.stock += request.amount
context.state('item').pack(item_state)
elif isinstance(request, StockRequest):
# If the item state is None we return an error
if item_state is None:
# Item does not exist yet. Return error.
if not request.internal:
response = ResponseMessage()
response.result = json.dumps({'result': 'not_found'})
else:
response = StockResponse()
response.item_id = request.subtract_stock.id
response.result = 'failure'
else:
# check which field we have
msg_type = request.WhichOneof('message')
#logger.debug(f'Got message of type {msg_type}')
if msg_type == "find_item":
response = ResponseMessage()
response.result = json.dumps(
{'id:': item_state.id, 'price': item_state.price, 'stock': item_state.stock})
context.state('item').pack(item_state)
elif msg_type == "subtract_stock":
new_amount = item_state.stock - request.subtract_stock.amount
if not request.internal:
response = ResponseMessage()
else:
response = StockResponse()
if new_amount >= 0:
item_state.stock -= request.subtract_stock.amount
context.state('item').pack(item_state)
if not request.internal:
response.result = json.dumps({'result': 'success', 'item_id': item_state.id})
else:
# Include the item id and price
response.price = item_state.price
response.item_id = item_state.id
response.result = 'success'
else:
if not request.internal:
response.result = json.dumps({'result': 'stock too low', 'item_id': item_state.id})
else:
response.price = item_state.price
response.item_id = item_state.id
response.result = 'failure'
elif msg_type == "add_stock":
item_state.stock += request.add_stock.amount
context.state('item').pack(item_state)
# send the response.
response = ResponseMessage()
response.result = json.dumps({'result': 'success', 'item_id': item_state.id})
if response:
# Use the same request id in the message body
# and use the request worker_id as key of the message
if not request.internal:
response.request_id = request.request_info.request_id
# create the egress message and send it to the
# users/out egress
egress_message = kafka_egress_record(
topic=STOCK_EVENTS_TOPIC,
key=request.request_info.worker_id,
value=response
)
context.pack_and_send_egress("stock/out", egress_message)
else:
response.request_info.request_id = request.request_info.request_id
response.request_info.worker_id = request.request_info.worker_id
context.pack_and_send("orders/order", str(request.order_id), response)
def pack_and_send_egress(self, topic, value):
any = Any()
any.Pack(value)
egress_message = kafka_egress_record(topic=topic, value=any)
self._context.pack_and_send_egress(self._egress_type_name, egress_message)
def operate_user(context,
request: typing.Union[UserPayRequest, UserCancelPayRequest,
UserRequest, CreateUserRequest]):
""" Does all the operations with a single user
Has the state of a user in a UserData() object that includes
its id and credit under the name 'user' """
# Get the current state for a given user
# could have to handle the state not existing for this user
state: UserData = context.state('user').unpack(UserData)
response = None
# ----------------------------------------
# Messages from the payment endpoint
# ----------------------------------------
if isinstance(request, UserPayRequest):
#logger.debug('Received request to decrement user credit')
# calculate if the credit is enough to pay for the product
# get the credit
response = UserPayResponse()
response.order_id = request.order_id
# copy the information of the request_info
response = copy_request_info(request, response)
# if the user exists then do the checks
if state:
# see whether we should return success or failure
if state.credit - request.amount < 0:
response.success = False
else:
state.credit -= request.amount
response.success = True
else:
response.success = False
# pack the state
context.state('user').pack(state)
# respond to the payment service
context.pack_and_reply(response)
return
elif isinstance(request, UserCancelPayRequest):
#logger.debug('Received request to cancel a payment')
# add the amount specified to the user credit
response = UserPayResponse()
response.order_id = request.order_id
# copy the information
response = copy_request_info(request, response)
if state:
state.credit += request.amount
# reply
response.success = True
else:
response.success = False
# pack the state
context.state('user').pack(state)
# reply to the sender function
context.pack_and_reply(response)
return
# -------------------------------------
# Interaction with the user endpoint
# -------------------------------------
elif isinstance(request, CreateUserRequest):
# we are given the uuid in the message so that's already done
state = UserData()
state.id = request.id
state.credit = 0
#logger.debug(f'Created new user with id {request.id}')
context.state('user').pack(state)
response = ResponseMessage()
response.result = json.dumps({'user_id': state.id})
elif isinstance(request, UserRequest):
# check which field we have
msg_type = request.WhichOneof('message')
#logger.debug(f'Got message of type {msg_type}')
# If the state is None we return an error
if not state:
response = ResponseMessage()
response.result = json.dumps(
{'result': 'failure: user does not exist'})
else:
# If the state exists we then operate with it
if msg_type == 'find_user':
response = ResponseMessage()
response.result = json.dumps({
'user_id': state.id,
'credit': state.credit
})
# pack the state
context.state('user').pack(state)
elif msg_type == 'remove_user':
del context['user']
response = ResponseMessage()
response.result = json.dumps({'result': 'success'})
elif msg_type == 'add_credit':
# Update the credit and save state
state.credit += request.add_credit.amount
context.state('user').pack(state)
# send the response
response = ResponseMessage()
response.result = json.dumps({'result': 'success'})
elif msg_type == 'subtract_credit':
# try to subtract the amount from the user credit
new_amount = state.credit - request.subtract_credit.amount
response = ResponseMessage()
if new_amount >= 0:
state.credit -= request.subtract_credit.amount
context.state('user').pack(state)
response.result = json.dumps({'result': 'success'})
else:
response.result = json.dumps({'result': 'failure'})
else:
logger.error('Received unknown message type!')
# respond if needed
if response:
# Use the same request id in the message body
# and use the request worker_id as key of the message
response.request_id = request.request_info.request_id
# create the egress message and send it to the
# users/out egress
egress_message = kafka_egress_record(
topic=USER_EVENTS_TOPIC,
key=request.request_info.worker_id,
value=response)
context.pack_and_send_egress("users/out", egress_message)
def transaction_manager(context, message: Union[Transaction, ReportedFraud, MerchantScore, FraudScore]):
"""
The transaction manager coordinates all communication between the various functions.
This includes building up the feature vector, calling into the model, and reporting
results to the user.
"""
if isinstance(message, Transaction):
context.state("transaction").pack(message)
context.pack_and_send(
"ververica/counter",
message.account,
QueryFraud())
context.pack_and_send(
"ververica/merchant",
message.merchant,
QueryMerchantScore())
elif isinstance(message, ReportedFraud):
m_score = context.state("merchant_score").unpack(MerchantScore)
if not m_score:
context.state("fraud_count").pack(message)
else:
transaction = context.state("transaction").unpack(Transaction)
vector = FeatureVector()
vector.fraud_count = message.count
vector.merchant_score = m_score.score
vector.amount = transaction.amount
context.pack_and_send(
"ververica/model",
transaction.account,
vector)
elif isinstance(message, MerchantScore):
count = context.state("fraud_count").unpack(ReportedFraud)
if not count:
context.state("merchant_score").pack(message)
else:
transaction = context.state("transaction").unpack(Transaction)
vector = FeatureVector()
vector.fraud_count = count.count
vector.merchant_score = message.score
vector.amount = transaction.amount
context.pack_and_send(
"ververica/model",
transaction.account,
vector)
elif isinstance(message, FraudScore):
if message.score > THRESHOLD:
transaction = context.state("transaction").unpack(Transaction)
egress_message = kafka_egress_record(topic="alerts", key=transaction.account, value=transaction)
context.pack_and_send_egress("ververica/kafka-egress", egress_message)
del context["transaction"]
del context["fraud_count"]
del context["merchant_score"]
