async def handle_transfer(context, message: Transfer, request_id: str) -> None: # Send messages subtract_credit = SubtractCredit(amount=message.amount) context.pack_and_send_atomic_invocation("ycsb-example/account_function", message.outgoing_id, subtract_credit) add_credit = AddCredit(amount=message.amount) context.pack_and_send_atomic_invocation("ycsb-example/account_function", message.incoming_id, add_credit) # Send on success response = Response(request_id=request_id, status_code=200) egress_message = kafka_egress_record(topic="responses", key=request_id, value=response) context.pack_and_send_egress_on_success("ycsb-example/kafka-egress", egress_message) # Send on failure response = Response(request_id=request_id, status_code=422) egress_message = kafka_egress_record(topic="responses", key=request_id, value=response) context.pack_and_send_egress_on_failure("ycsb-example/kafka-egress", egress_message) # Send on retryable (e.g. deadlock) response = Response(request_id=request_id, status_code=401) egress_message = kafka_egress_record(topic="responses", key=request_id, value=response) context.pack_and_send_egress_on_retryable("ycsb-example/kafka-egress", egress_message)
def transfer_function(context, request: Transfer): # Send messages delete = Delete() context.pack_and_send_atomic_invocation("ycsb-example/account_function", request.outgoing_id, delete) add_credit = AddCredit(amount=request.amount) context.pack_and_send_atomic_invocation("ycsb-example/account_function", request.incoming_id, add_credit) # Send on success response = Response(request_id=context.address.identity, status_code=200) egress_message = kafka_egress_record(topic="responses", key=context.address.identity, value=response) context.pack_and_send_egress("ycsb-example/kafka-egress", egress_message, success=True) # Send on failure response = Response(request_id=context.address.identity, status_code=422) egress_message = kafka_egress_record(topic="responses", key=context.address.identity, value=response) context.pack_and_send_egress("ycsb-example/kafka-egress", egress_message, success=False)
def egress(context, message): # send a message to an external system via an egress. Egresses needs to be defined in a module.yaml # and can be referenced by type. # The following two lines prepare a message to send to the pre-built Kafka egress. key = context.address.identity # use the identity part of our own address as the target Kafka key. record = kafka_egress_record(topic="events", key=key, value=Event()) context.pack_and_send_egress("walkthrough/events-egress", record)
def overview(context, state: OrderStateWithPrevious): if state.status == "UNASSIGNED": no_unassigned = context.state('no_unassigned').unpack(NoState) if not no_unassigned: no_unassigned = NoState() no_unassigned.counter = 1 else: no_unassigned.counter += 1 context.state('no_unassigned').pack(no_unassigned) elif state.status == "ASSIGNED": no_unassigned = context.state('no_unassigned').unpack(NoState) if no_unassigned and state.previous: no_unassigned.counter -= 1 context.state('no_unassigned').pack(no_unassigned) no_assigned = context.state('no_assigned').unpack(NoState) if not no_assigned: no_assigned = NoState() no_assigned.counter = 1 else: no_assigned.counter += 1 context.state('no_assigned').pack(no_assigned) elif state.status == "IN_PROGRESS": no_assigned = context.state('no_assigned').unpack(NoState) if no_assigned and state.previous: no_assigned.counter -= 1 context.state('no_assigned').pack(no_assigned) no_in_progress = context.state('no_in_progress').unpack(NoState) if not no_in_progress: no_in_progress = NoState() no_in_progress.counter = 1 else: no_in_progress.counter += 1 context.state('no_in_progress').pack(no_in_progress) elif state.status == "DELIVERED": no_in_progress = context.state('no_in_progress').unpack(NoState) if no_in_progress and state.previous: no_in_progress.counter -= 1 context.state('no_in_progress').pack(no_in_progress) no_delivered = context.state('no_delivered').unpack(NoState) if not no_delivered: no_delivered = NoState() no_delivered.counter = 1 else: no_delivered.counter += 1 context.state('no_delivered').pack(no_delivered) overview = compute_overview(context) egress_message = kafka_egress_record(topic="overviews", key="overview", value=overview) context.pack_and_send_egress("lieferbot/status", egress_message)
def send_response(context, response_message, worker_id): #logger.debug(worker_id) egress_message = kafka_egress_record(topic=PAYMENT_EVENTS_TOPIC, key=worker_id, value=response_message) #logger.info(f'Sending message {egress_message}') context.pack_and_send_egress("payments/out", egress_message)
def forward_to_egress(context, invoke_result: InvokeResult): """ Simply forwards the results to the Kafka egress. """ egress_message = kafka_egress_record(topic="invoke-results", key=invoke_result.id, value=invoke_result) context.pack_and_send_egress( "org.apache.flink.statefun.e2e.remote/invoke-results", egress_message)
def score(context, msg: ServerMetricHistory): print(msg) if not repair(msg): return request = RepairServer() request.server_id = context.address.identity message = kafka_egress_record(topic="repair", key=request.server_id, value=request) print("fixing " + request.server_id) context.pack_and_send_egress("io/repairs", message)
def monitor(context, order_update: OrderUpdate): state = context.state('order_state').unpack(OrderState) state_with_previous = OrderStateWithPrevious() state_with_previous.status = order_update.status if not state: state = OrderState() state_with_previous.previous = False else: state_with_previous.previous = True state.status = order_update.status context.state('order_state').pack(state) if state.status == "UNASSIGNED": time_unassigned = context.state('time_unassigned').unpack(Time) if not time_unassigned: time_unassigned = Time() time_unassigned.time = order_update.time context.state('time_unassigned').pack(time_unassigned) elif state.status == "ASSIGNED": time_assigned = context.state('time_assigned').unpack(Time) if not time_assigned: time_assigned = Time() time_assigned.time = order_update.time context.state('time_assigned').pack(time_assigned) elif state.status == "IN_PROGRESS": time_in_progress = context.state('time_in_progress').unpack(Time) if not time_in_progress: time_in_progress = Time() time_in_progress.time = order_update.time context.state('time_in_progress').pack(time_in_progress) elif state.status == "DELIVERED": time_delivered = context.state('time_delivered').unpack(Time) if not time_delivered: time_delivered = Time() time_delivered.time = order_update.time context.state('time_delivered').pack(time_delivered) report = compute_report(context, order_update) egress_message = kafka_egress_record(topic="reports", key=order_update.id, value=report) context.pack_and_send_egress("lieferbot/status", egress_message) context.pack_and_send("lieferbot/overview", "overview", state_with_previous) context.pack_and_send("lieferbot/timeout_counter", order_update.id, order_update) context.pack_and_send("lieferbot/timeout_check", order_update.id, state)
def fun(context, message): # state access seen = context.state('seen').unpack(SeenCount) seen.seen += 1 context.state('seen').pack(seen) # regular state access seenAny = context['seen'] seenAny.Unpack(seen) # sending and replying context.pack_and_reply(seen) any = Any() any.type_url = 'type.googleapis.com/k8s.demo.SeenCount' context.send("bar.baz/foo", "12345", any) # delayed messages context.send_after(timedelta(hours=1), "night/owl", "1", any) # egresses context.send_egress("foo.bar.baz/my-egress", any) context.pack_and_send_egress("foo.bar.baz/my-egress", seen) # kafka egress context.pack_and_send_egress("sdk/kafka", kafka_egress_record(topic="hello", key=u"hello world", value=seen)) context.pack_and_send_egress("sdk/kafka", kafka_egress_record(topic="hello", value=seen)) # AWS Kinesis generic egress context.pack_and_send_egress("sdk/kinesis", kinesis_egress_record( stream="hello", partition_key=u"hello world", value=seen, explicit_hash_key=u"1234")) context.pack_and_send_egress("sdk/kinesis", kinesis_egress_record( stream="hello", partition_key=u"hello world", value=seen))
def greet(context, message: LoginEvent): state = context.state('seen_count').unpack(SeenCount) if not state: state = SeenCount() state.seen = 1 else: state.seen += 1 context.state('seen_count').pack(state) egress_message = kafka_egress_record(topic="seen", key=message.user_name, value=state) context.pack_and_send_egress("k8s-demo/greets-egress", egress_message)
async def send_response(context, request_id: str, status_code: int, message=None) -> None: response = Response(request_id=request_id, status_code=status_code) if message: out = Any() out.Pack(message) response.message.CopyFrom(out) egress_message = kafka_egress_record(topic="responses", key=request_id, value=response) context.pack_and_send_egress("ycsb-example/kafka-egress", egress_message)
def operate_order(context, msg: typing.Union[CreateOrder, OrderRequest, OrdersPayFind, OrderPaymentCancel, PaymentStatus, StockResponse]): """ Does all the operations with a single order """ response = None if isinstance(msg, CreateOrder): response = create_order_with_id(context, msg) elif isinstance(msg, OrderRequest): msg_type = msg.WhichOneof('message') # logger.debug(f'Got message of type {msg_type}') if msg_type == 'remove_order': response = remove_order(context, msg) elif msg_type == 'find_order': response = find_order(context, msg) elif msg_type == 'add_item': response = add_item(context, msg) elif msg_type == 'remove_item': response = remove_item(context, msg) elif msg_type == 'order_checkout': response = order_checkout(context, msg) elif isinstance(msg, OrdersPayFind): order_payment_find(context, msg) elif isinstance(msg, OrderPaymentCancel): order_payment_cancel(context, msg) elif isinstance(msg, PaymentStatus): response = order_payment_confirm(context, msg) elif isinstance(msg, StockResponse): response = order_add_item_reply(context, msg) else: logger.error('Received unknown message type!') if response: response.request_id = msg.request_info.request_id egress_message = kafka_egress_record(topic=ORDER_EVENTS_TOPIC, key=msg.request_info.worker_id, value=response) context.pack_and_send_egress("orders/out", egress_message)
def greet(context, greet_request: GreetRequest): state = context.state('seen_count').unpack(SeenCount) if not state: state = SeenCount() state.seen = 1 else: state.seen += 1 context.state('seen_count').pack(state) response = compute_greeting(greet_request.name, state.seen) egress_message = kafka_egress_record(topic="greetings", key=greet_request.name, value=response) context.pack_and_send_egress("example/greets", egress_message)
def check_request(context, transfer_request: TransferRequest): state = context.state('amount').unpack(AmountTransfered) if not state: state = AmountTransfered() state.amount = transfer_request.amount else: state.amount += transfer_request.amount context.state('amount').pack(state) response = send_acknowledgement(transfer_request.number, transfer_request.amount, state.amount) egress_message = kafka_egress_record(topic="transfer_response", key=transfer_request.number, value=response) context.pack_and_send_egress("demo/transfer/acknowledgement", egress_message)
async def account_function(context, request: Wrapper): # Get state state = context.state('state').unpack(State) # messages from outside request_id = request.request_id message = request.message if message.Is(Transfer.DESCRIPTOR): transfer = Transfer() message.Unpack(transfer) read = Read() read.id = transfer.outgoing_id wrapped = Wrapper() wrapped.request_id = request_id outgoing_message = Any() outgoing_message.Pack(read) wrapped.message.CopyFrom(outgoing_message) context.pack_and_send("ycsb-example/account_function", transfer.outgoing_id, wrapped) elif message.Is(Read.DESCRIPTOR): read = Read() message.Unpack(read) update = Update() wrapped = Wrapper() wrapped.request_id = request_id outgoing_message = Any() outgoing_message.Pack(update) wrapped.message.CopyFrom(outgoing_message) context.pack_and_reply(wrapped) elif message.Is(Update.DESCRIPTOR): update = Update() message.Unpack(update) response = Response(request_id=request_id, status_code=200) egress_message = kafka_egress_record(topic="responses", key=request_id, value=response) context.pack_and_send_egress("ycsb-example/kafka-egress", egress_message) else: print("Test test test")
def greet(context, greet_request: GreetRequest): state = context.state('seen_count').unpack(SeenCount) if not state: state = SeenCount() state.seen = 1 state.sum = 0 else: state.seen += 1 state.sum += greet_request.val context.state('seen_count').pack(state) response = compute_greeting(greet_request.name, state.seen) response.val = greet_request.val egress_message = kafka_egress_record(topic="greetings", key="vals".encode('utf-8'), value=state) context.pack_and_send_egress("example/greets", egress_message)
def timeout(context, request: typing.Union[OrderState, OrderUpdate]): if isinstance(request, OrderState): state = context.state('order_state').unpack(OrderState) if not state: state = OrderState() state.status = request.status context.state('order_state').pack(state) elif isinstance(request, OrderUpdate): state = context.state('order_state').unpack(OrderState) if (state.status == request.status): report = TimeoutReport() report.order.status = state.status report.orderId = request.id egress_message = kafka_egress_record(topic="timeouts", key=request.id, value=report) context.pack_and_send_egress("lieferbot/status", egress_message)
def manage_stock(context, request: typing.Union[StockRequest, CreateItemRequest, OrderAddItemStockRequest]): # Get the current state. item_state: ItemData = context.state('item').unpack(ItemData) if isinstance(request, CreateItemRequest): item_state = ItemData() item_state.id = request.id item_state.price = request.price item_state.stock = 0 context.state('item').pack(item_state) #logger.debug(f'Created new item with id {request.id}') response = ResponseMessage() response.result = json.dumps({'item_id': item_state.id}) elif isinstance(request, OrderAddItemStockRequest): response = None if item_state is None: pass else: item_state.stock += request.amount context.state('item').pack(item_state) elif isinstance(request, StockRequest): # If the item state is None we return an error if item_state is None: # Item does not exist yet. Return error. if not request.internal: response = ResponseMessage() response.result = json.dumps({'result': 'not_found'}) else: response = StockResponse() response.item_id = request.subtract_stock.id response.result = 'failure' else: # check which field we have msg_type = request.WhichOneof('message') #logger.debug(f'Got message of type {msg_type}') if msg_type == "find_item": response = ResponseMessage() response.result = json.dumps( {'id:': item_state.id, 'price': item_state.price, 'stock': item_state.stock}) context.state('item').pack(item_state) elif msg_type == "subtract_stock": new_amount = item_state.stock - request.subtract_stock.amount if not request.internal: response = ResponseMessage() else: response = StockResponse() if new_amount >= 0: item_state.stock -= request.subtract_stock.amount context.state('item').pack(item_state) if not request.internal: response.result = json.dumps({'result': 'success', 'item_id': item_state.id}) else: # Include the item id and price response.price = item_state.price response.item_id = item_state.id response.result = 'success' else: if not request.internal: response.result = json.dumps({'result': 'stock too low', 'item_id': item_state.id}) else: response.price = item_state.price response.item_id = item_state.id response.result = 'failure' elif msg_type == "add_stock": item_state.stock += request.add_stock.amount context.state('item').pack(item_state) # send the response. response = ResponseMessage() response.result = json.dumps({'result': 'success', 'item_id': item_state.id}) if response: # Use the same request id in the message body # and use the request worker_id as key of the message if not request.internal: response.request_id = request.request_info.request_id # create the egress message and send it to the # users/out egress egress_message = kafka_egress_record( topic=STOCK_EVENTS_TOPIC, key=request.request_info.worker_id, value=response ) context.pack_and_send_egress("stock/out", egress_message) else: response.request_info.request_id = request.request_info.request_id response.request_info.worker_id = request.request_info.worker_id context.pack_and_send("orders/order", str(request.order_id), response)
def pack_and_send_egress(self, topic, value): any = Any() any.Pack(value) egress_message = kafka_egress_record(topic=topic, value=any) self._context.pack_and_send_egress(self._egress_type_name, egress_message)
def operate_user(context, request: typing.Union[UserPayRequest, UserCancelPayRequest, UserRequest, CreateUserRequest]): """ Does all the operations with a single user Has the state of a user in a UserData() object that includes its id and credit under the name 'user' """ # Get the current state for a given user # could have to handle the state not existing for this user state: UserData = context.state('user').unpack(UserData) response = None # ---------------------------------------- # Messages from the payment endpoint # ---------------------------------------- if isinstance(request, UserPayRequest): #logger.debug('Received request to decrement user credit') # calculate if the credit is enough to pay for the product # get the credit response = UserPayResponse() response.order_id = request.order_id # copy the information of the request_info response = copy_request_info(request, response) # if the user exists then do the checks if state: # see whether we should return success or failure if state.credit - request.amount < 0: response.success = False else: state.credit -= request.amount response.success = True else: response.success = False # pack the state context.state('user').pack(state) # respond to the payment service context.pack_and_reply(response) return elif isinstance(request, UserCancelPayRequest): #logger.debug('Received request to cancel a payment') # add the amount specified to the user credit response = UserPayResponse() response.order_id = request.order_id # copy the information response = copy_request_info(request, response) if state: state.credit += request.amount # reply response.success = True else: response.success = False # pack the state context.state('user').pack(state) # reply to the sender function context.pack_and_reply(response) return # ------------------------------------- # Interaction with the user endpoint # ------------------------------------- elif isinstance(request, CreateUserRequest): # we are given the uuid in the message so that's already done state = UserData() state.id = request.id state.credit = 0 #logger.debug(f'Created new user with id {request.id}') context.state('user').pack(state) response = ResponseMessage() response.result = json.dumps({'user_id': state.id}) elif isinstance(request, UserRequest): # check which field we have msg_type = request.WhichOneof('message') #logger.debug(f'Got message of type {msg_type}') # If the state is None we return an error if not state: response = ResponseMessage() response.result = json.dumps( {'result': 'failure: user does not exist'}) else: # If the state exists we then operate with it if msg_type == 'find_user': response = ResponseMessage() response.result = json.dumps({ 'user_id': state.id, 'credit': state.credit }) # pack the state context.state('user').pack(state) elif msg_type == 'remove_user': del context['user'] response = ResponseMessage() response.result = json.dumps({'result': 'success'}) elif msg_type == 'add_credit': # Update the credit and save state state.credit += request.add_credit.amount context.state('user').pack(state) # send the response response = ResponseMessage() response.result = json.dumps({'result': 'success'}) elif msg_type == 'subtract_credit': # try to subtract the amount from the user credit new_amount = state.credit - request.subtract_credit.amount response = ResponseMessage() if new_amount >= 0: state.credit -= request.subtract_credit.amount context.state('user').pack(state) response.result = json.dumps({'result': 'success'}) else: response.result = json.dumps({'result': 'failure'}) else: logger.error('Received unknown message type!') # respond if needed if response: # Use the same request id in the message body # and use the request worker_id as key of the message response.request_id = request.request_info.request_id # create the egress message and send it to the # users/out egress egress_message = kafka_egress_record( topic=USER_EVENTS_TOPIC, key=request.request_info.worker_id, value=response) context.pack_and_send_egress("users/out", egress_message)
def transaction_manager(context, message: Union[Transaction, ReportedFraud, MerchantScore, FraudScore]): """ The transaction manager coordinates all communication between the various functions. This includes building up the feature vector, calling into the model, and reporting results to the user. """ if isinstance(message, Transaction): context.state("transaction").pack(message) context.pack_and_send( "ververica/counter", message.account, QueryFraud()) context.pack_and_send( "ververica/merchant", message.merchant, QueryMerchantScore()) elif isinstance(message, ReportedFraud): m_score = context.state("merchant_score").unpack(MerchantScore) if not m_score: context.state("fraud_count").pack(message) else: transaction = context.state("transaction").unpack(Transaction) vector = FeatureVector() vector.fraud_count = message.count vector.merchant_score = m_score.score vector.amount = transaction.amount context.pack_and_send( "ververica/model", transaction.account, vector) elif isinstance(message, MerchantScore): count = context.state("fraud_count").unpack(ReportedFraud) if not count: context.state("merchant_score").pack(message) else: transaction = context.state("transaction").unpack(Transaction) vector = FeatureVector() vector.fraud_count = count.count vector.merchant_score = message.score vector.amount = transaction.amount context.pack_and_send( "ververica/model", transaction.account, vector) elif isinstance(message, FraudScore): if message.score > THRESHOLD: transaction = context.state("transaction").unpack(Transaction) egress_message = kafka_egress_record(topic="alerts", key=transaction.account, value=transaction) context.pack_and_send_egress("ververica/kafka-egress", egress_message) del context["transaction"] del context["fraud_count"] del context["merchant_score"]