def main(): platform = Platform() platform.connect_to_cloud() # wait until connected while not platform.cloud_ready(): try: time.sleep(0.1) except KeyboardInterrupt: # this allows us to stop the application while waiting for cloud connection platform.stop() return util = Utility() pubsub = PubSub() if pubsub.feature_enabled(): # Publish an event # headers = optional parameters for the event # body = event payload for x in range(10): print("publishing event#", x) pubsub.publish("hello.topic", headers={ "some_parameter": "some_value", "n": x }, body="hello world " + util.get_iso_8601(time.time())) # quit application platform.stop() else: print("Pub/Sub feature not available from the underlying event stream") print("Did you start the language connector with Kafka?") print( "e.g. java -Dcloud.connector=kafka -Dcloud.services=kafka.reporter -jar language-connector-1.12.31.jar" )
def main(): platform = Platform() log = platform.get_logger() # connect to the network platform.connect_to_cloud() # wait until connected while not platform.cloud_ready(): try: time.sleep(0.1) except KeyboardInterrupt: # this allows us to stop the application while waiting for cloud connection platform.stop() return # # You can create a new I/O stream using ObjectStreamIO. # This requires a live connection to the language connector. # stream = ObjectStreamIO(10) in_stream_id = stream.get_input_stream() out_stream_id = stream.get_output_stream() output_stream = ObjectStreamWriter(out_stream_id) input_stream = ObjectStreamReader(in_stream_id) for i in range(100): output_stream.write('hello world ' + str(i)) # # if output stream is not closed, input will timeout # Therefore, please use try-except for TimeoutError in the iterator for-loop below. # output_stream.close() for block in input_stream.read(5.0): if block is None: log.info("EOF") else: log.info(block) input_stream.close() # # this will keep the main thread running in the background # so we can use Control-C or KILL signal to stop the application platform.run_forever()
def hello(headers: dict, body: any, instance: int): # regular function signature (headers: dict, body: any, instance: int) Platform().log.info("#"+str(instance)+" "+str(headers)+" body="+str(body)) # to set status, headers and body, return them in an event envelope result = EventEnvelope().set_header('hello', 'world').set_body(body) for h in headers: result.set_header(h, headers[h]) return result
def main(): platform = Platform() # register a function platform.register('hello.world', hello, 10) po = PostOffice() # demonstrate sending asynchronously. Note that key-values in the headers will be encoded as strings po.send('hello.world', headers={'one': 1}, body='hello world one') po.send('hello.world', headers={'two': 2}, body='hello world two') # demonstrate a RPC request try: result = po.request('hello.world', 2.0, headers={'some_key': 'some_value'}, body='hello world') if isinstance(result, EventEnvelope): print('Received RPC response:') print("HEADERS =", result.get_headers(), ", BODY =", result.get_body(), ", STATUS =", result.get_status(), ", EXEC =", result.get_exec_time(), ", ROUND TRIP =", result.get_round_trip(), "ms") except TimeoutError as e: print("Exception: ", str(e)) # demonstrate drop-n-forget for n in range(20): po.send('hello.world', body='just a drop-n-forget message ' + str(n)) # # this will keep the main thread running in the background # so we can use Control-C or KILL signal to stop the application platform.run_forever()
def __init__(self): self.platform = Platform() self.po = PostOffice() self.util = Utility() self.subscription = dict() def subscription_sync(headers: dict, body: any): if 'type' in headers and headers['type'] == 'subscription_sync': if len(self.subscription) > 0: for topic in self.subscription: route_map = self.subscription[topic] for route in route_map: parameters = route_map[route] self.platform.log.info('Update subscription ' + topic + ' -> ' + route) self.subscribe(topic, route, parameters) else: self.platform.log.info('No subscription to update') self.platform.register('pub.sub.sync', subscription_sync, 1, is_private=True)
def __init__(self, route: str = None, expiry_seconds: int = 1800): self.platform = Platform() self.po = PostOffice() self.util = Utility() self.route = None self.input_stream = None self.output_stream = None self.eof = False self.input_closed = False self.output_closed = False if route is not None: # open an existing stream if isinstance(route, str): name: str = route if name.startswith('stream.') and '@' in name: self.route = name if self.route is None: raise ValueError('Invalid stream route') else: # create a new stream if not isinstance(expiry_seconds, int): raise ValueError('expiry_seconds must be int') result = self.po.request(self.STREAM_IO_MANAGER, 6.0, headers={ 'type': 'create', 'expiry_seconds': expiry_seconds }) if isinstance(result, EventEnvelope) and isinstance(result.get_body(), str) \ and result.get_status() == 200: name: str = result.get_body() if name.startswith('stream.') and '@' in name: self.route = name if self.route is None: raise IOError('Stream manager is not responding correctly')
class PostOffice: """ Convenient class for making RPC, async and callback. """ def __init__(self): self.platform = Platform() self.util = Utility() def get_route(self): """ Obtain my route name for the currently running service. This is useful for Role Based Access Control (RBAC) to restrict certain user roles for a service. Note that RBAC is the responsibility of the user application. Returns: route name """ trace_info = self.get_trace() return "?" if trace_info is None else trace_info.get_route() def get_trace_id(self): return self.platform.get_trace_id() def get_trace(self): return self.platform.get_trace() def annotate_trace(self, key: str, value: str): self.platform.annotate_trace(key, value) def broadcast(self, route: str, headers: dict = None, body: any = None) -> None: self.util.validate_service_name(route) if headers is None and body is None: raise ValueError( 'Unable to broadcast because both headers and body are missing' ) event = EventEnvelope().set_to(route) if headers is not None: if not isinstance(headers, dict): raise ValueError('headers must be dict') for h in headers: event.set_header(h, str(headers[h])) if body is not None: event.set_body(body) self.platform.send_event(event, True) def send_later(self, route: str, headers: dict = None, body: any = None, reply_to: str = None, me=True, seconds: float = 1.0) -> None: self.util.validate_service_name(route, True) if isinstance(seconds, float) or isinstance(seconds, int): if seconds > 0: if headers is None and body is None: raise ValueError( 'Unable to send because both headers and body are missing' ) event = EventEnvelope().set_to(route) if headers is not None: if not isinstance(headers, dict): raise ValueError('headers must be dict') for h in headers: event.set_header(str(h), str(headers[h])) if body is not None: event.set_body(body) if reply_to is not None: if not isinstance(reply_to, str): raise ValueError('reply_to must be str') # encode 'me' in the "call back" if replying to this instance event.set_reply_to(reply_to, me) self.platform.send_event_later(event, seconds) else: raise ValueError('delay in seconds must be larger than zero') else: raise ValueError('delay in seconds must be int or float') def send_event_later(self, event: EventEnvelope, seconds: float = 1.0, me=True): if event.get_reply_to() is not None: event.set_reply_to(event.get_reply_to(), me) self.platform.send_event_later(event, seconds) def send(self, route: str, headers: dict = None, body: any = None, reply_to: str = None, me=True) -> None: self.util.validate_service_name(route, True) if headers is None and body is None: raise ValueError( 'Unable to send because both headers and body are missing') event = EventEnvelope().set_to(route) if headers is not None: if not isinstance(headers, dict): raise ValueError('headers must be dict') for h in headers: event.set_header(str(h), str(headers[h])) if body is not None: event.set_body(body) if reply_to is not None: if not isinstance(reply_to, str): raise ValueError('reply_to must be str') # encode 'me' in the "call back" if replying to this instance event.set_reply_to(reply_to, me) self.platform.send_event(event) def send_event(self, event: EventEnvelope, me=True): if event.get_reply_to() is not None: event.set_reply_to(event.get_reply_to(), me) self.platform.send_event(event) def request(self, route: str, timeout_seconds: float, headers: dict = None, body: any = None, correlation_id: str = None) -> EventEnvelope: self.util.validate_service_name(route, True) if headers is None and body is None: raise ValueError( 'Unable to make RPC call because both headers and body are missing' ) timeout_value = self.util.get_float(timeout_seconds) if timeout_value <= 0: raise ValueError( 'timeout value in seconds must be positive number') event = EventEnvelope().set_to(route) if headers is not None: if not isinstance(headers, dict): raise ValueError('headers must be dict') for h in headers: event.set_header(h, str(headers[h])) if body is not None: event.set_body(body) if correlation_id is not None: event.set_correlation_id(str(correlation_id)) response = self.platform.request(event, timeout_seconds) if isinstance(response, EventEnvelope): if response.get_tag('exception') is None: return response else: raise AppException(response.get_status(), response.get_body()) raise ValueError( f'Expect response is EventEnvelope, actual: ({response})') def single_request(self, event: EventEnvelope, timeout_seconds: float): response = self.platform.request(event, timeout_seconds) if isinstance(response, EventEnvelope): if response.get_tag('exception') is None: return response else: raise AppException(response.get_status(), response.get_body()) raise ValueError( f'Expect response is EventEnvelope, actual: ({response})') def parallel_request(self, events: list, timeout_seconds: float) -> list: return self.platform.parallel_request(events, timeout_seconds) def exists(self, routes: any): return self.platform.exists(routes)
def main(): platform = Platform() # you can register a method of a class platform.register('hello.world.1', Hi().hello, 5) # or register a function platform.register('hello.world.2', hello, 10) po = PostOffice() # demonstrate sending asynchronously. Note that key-values in the headers will be encoded as strings po.send('hello.world.1', headers={'one': 1}, body='hello world one') po.send('hello.world.2', headers={'two': 2}, body='hello world two') # demonstrate a RPC request try: result = po.request('hello.world.2', 2.0, headers={'some_key': 'some_value'}, body='hello world') if isinstance(result, EventEnvelope): print('Received RPC response:') print("HEADERS =", result.get_headers(), ", BODY =", result.get_body(), ", STATUS =", result.get_status(), ", EXEC =", result.get_exec_time(), ", ROUND TRIP =", result.get_round_trip(), "ms") except TimeoutError as e: print("Exception: ", str(e)) # illustrate parallel RPC requests event_list = list() event_list.append( EventEnvelope().set_to('hello.world.1').set_body("first request")) event_list.append( EventEnvelope().set_to('hello.world.2').set_body("second request")) try: result = po.parallel_request(event_list, 2.0) if isinstance(result, list): print('Received', len(result), 'RPC responses:') for res in result: print("HEADERS =", res.get_headers(), ", BODY =", res.get_body(), ", STATUS =", res.get_status(), ", EXEC =", res.get_exec_time(), ", ROUND TRIP =", res.get_round_trip(), "ms") except TimeoutError as e: print("Exception: ", str(e)) # connect to the network platform.connect_to_cloud() # wait until connected while not platform.cloud_ready(): try: time.sleep(0.1) except KeyboardInterrupt: # this allows us to stop the application while waiting for cloud connection platform.stop() return # Demonstrate broadcast feature # the event will be broadcast to multiple application instances that serve the same route po.broadcast("hello.world.1", body="this is a broadcast message from " + platform.get_origin()) # demonstrate deferred delivery po.send_later('hello.world.1', headers={'hello': 'world'}, body='this message arrives 5 seconds later', seconds=5.0) # # this will keep the main thread running in the background # so we can use Control-C or KILL signal to stop the application platform.run_forever()
def hello(self, headers: dict, body: any): # singleton function signature (headers: dict, body: any) Platform().get_logger().info(self.MY_NAME + " " + str(headers) + ", " + str(body)) return body
def main(): platform = Platform() # this shows that we can register a route name for a function platform.register('hello.world', hello, 10) # Once it connects to the network, it is ready to serve requests platform.connect_to_cloud() # wait until connected while not platform.cloud_ready(): try: time.sleep(0.1) except KeyboardInterrupt: # this allows us to stop the application while waiting for cloud connection platform.stop() return # # this will keep the main thread running in the background # so we can use Control-C or KILL signal to stop the application platform.run_forever()
class PubSub: def __init__(self): self.platform = Platform() self.po = PostOffice() self.util = Utility() self.subscription = dict() def subscription_sync(headers: dict, body: any): if 'type' in headers and headers['type'] == 'subscription_sync': if len(self.subscription) > 0: for topic in self.subscription: route_map = self.subscription[topic] for route in route_map: parameters = route_map[route] self.platform.log.info('Update subscription ' + topic + ' -> ' + route) self.subscribe(topic, route, parameters) else: self.platform.log.info('No subscription to update') self.platform.register('pub.sub.sync', subscription_sync, 1, is_private=True) def feature_enabled(self): result = self.po.request('pub.sub.controller', 10.0, headers={'type': 'feature'}) return self._normalize_result(result, True) def list_topics(self): result = self.po.request('pub.sub.controller', 10.0, headers={'type': 'list'}) return self._normalize_result(result, list()) def exists(self, topic: str): if isinstance(topic, str): result = self.po.request('pub.sub.controller', 10.0, headers={ 'type': 'exists', 'topic': topic }) return self._normalize_result(result, True) else: return False def create_topic(self, topic: str): if isinstance(topic, str): result = self.po.request('pub.sub.controller', 10.0, headers={ 'type': 'create', 'topic': topic }) return self._normalize_result(result, True) else: raise ValueError("topic must be str") def delete_topic(self, topic: str): if isinstance(topic, str): result = self.po.request('pub.sub.controller', 10.0, headers={ 'type': 'delete', 'topic': topic }) return self._normalize_result(result, True) else: raise ValueError("topic must be str") def publish(self, topic: str, headers: dict = None, body: any = None): if isinstance(topic, str): # encode payload payload = dict() payload['body'] = body payload['headers'] = self._normalize_headers(headers) result = self.po.request('pub.sub.controller', 10.0, headers={ 'type': 'publish', 'topic': topic }, body=payload) return self._normalize_result(result, True) else: raise ValueError("topic must be str") def subscribe(self, topic: str, route: str, parameters: list = None): if isinstance(topic, str) and isinstance(route, str): if self.platform.has_route(route): normalized_config = self._normalize_parameters(parameters) result = self.po.request('pub.sub.controller', 10.0, body=normalized_config, headers={ 'type': 'subscribe', 'topic': topic, 'route': route }) done = self._normalize_result(result, True) if done: if topic not in self.subscription: self.subscription[topic] = dict() self.platform.log.info('Subscribed topic ' + topic) route_map: dict = self.subscription[topic] if route not in route_map: route_map[route] = normalized_config self.platform.log.info('Adding ' + route + ' to topic ' + topic) return done else: raise ValueError("Unable to subscribe topic " + topic + " because route " + route + " not registered") else: raise ValueError("topic and route must be str") def unsubscribe(self, topic: str, route: str): if isinstance(topic, str) and isinstance(route, str): if self.platform.has_route(route): result = self.po.request('pub.sub.controller', 10.0, headers={ 'type': 'unsubscribe', 'topic': topic, 'route': route }) done = self._normalize_result(result, True) if done: if topic in self.subscription: route_map: dict = self.subscription[topic] if route in route_map: route_map.pop(route) self.platform.log.info('Removing ' + route + ' from topic ' + topic) if len(route_map) == 0: self.subscription.pop(topic) self.platform.log.info('Unsubscribed topic ' + topic) return done else: raise ValueError("Unable to unsubscribe topic " + topic + " because route " + route + " not registered") else: raise ValueError("topic and route must be str") @staticmethod def _normalize_result(result: EventEnvelope, result_obj: any): if isinstance(result, EventEnvelope): if result.get_status() == 200: if isinstance(result.get_body(), type(result_obj)): return result.get_body() else: raise AppException(500, str(result.get_body())) else: raise AppException(result.get_status(), str(result.get_body())) @staticmethod def _normalize_headers(headers: dict): if headers is None: return dict() if isinstance(headers, dict): result = dict() for h in headers: result[str(h)] = str(headers[h]) return result else: raise ValueError("headers must be dict of str key-values") @staticmethod def _normalize_parameters(parameters: list): if parameters is None: return list() if isinstance(parameters, list): result = list() for h in parameters: result.append(str(h)) return result else: raise ValueError("headers must be a list of str")
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from mercury.platform import Platform from mercury.system.models import EventEnvelope from mercury.system.po import PostOffice platform = Platform() log = platform.get_logger() po = PostOffice() def hello(headers: dict, body: any, instance: int): # regular function signature (headers: dict, body: any, instance: int) log.info(f'#{instance} got header={headers} body={body}') # as a demo, just echo the original payload return body def main(): # register a function platform.register('hello.world', hello, 10)
def main(): platform = Platform() # we should register the custom trace processor as a singleton platform.register('distributed.trace.processor', my_trace_processor, 1) # Once it connects to the network, it is ready to serve requests platform.connect_to_cloud() # wait until connected while not platform.cloud_ready(): try: time.sleep(0.1) except KeyboardInterrupt: # this allows us to stop the application while waiting for cloud connection platform.stop() return # # this will keep the main thread running in the background # so we can use Control-C or KILL signal to stop the application platform.run_forever()
def main(): platform = Platform() # register a route name for a pub/sub subscriber function # setting number of instance to 1 because pub/sub subscriber is always a singleton platform.register('hello.world', hello, 1) # Once it connects to the network, it is ready to serve requests platform.connect_to_cloud() # wait until connected while not platform.cloud_ready(): try: time.sleep(0.1) except KeyboardInterrupt: # this allows us to stop the application while waiting for cloud connection platform.stop() return pubsub = PubSub() if pubsub.feature_enabled(): # # the pub/sub topic name must be different from the subscriber function name # # Note: # For kafka, the parameter list includes the following: # client_id, group_id and optional offset number (as a string) # e.g. ["client1", "group1"] or ["client1", "group1", "0"] # # In this example, it is reading from the beginning of the topic. # For a real application, it should read without the offset so that it can fetch the latest events. # pubsub.subscribe("hello.topic", "hello.world", ["client1", "group1", "0"]) else: print("Pub/Sub feature not available from the underlying event stream") print("Did you start the language connector with Kafka?") print( "e.g. java -Dcloud.connector=kafka -Dcloud.services=kafka.reporter -jar language-connector-1.12.31.jar" ) # # this will keep the main thread running in the background # so we can use Control-C or KILL signal to stop the application platform.run_forever()
class PostOffice: """ Convenient class for making RPC, async and callback. """ DEFERRED_DELIVERY = 'system.deferred.delivery' def __init__(self): self.platform = Platform() self.util = Utility() def get_route(self): """ Obtain my route name for the currently running service. This is useful for Role Based Access Control (RBAC) to restrict certain user roles for a service. Note that RBAC is the responsibility of the user application. :return: route name """ trace_info = self.get_trace() return "?" if trace_info is None else trace_info.get_route() def get_trace_id(self): return self.platform.get_trace_id() def get_trace(self): return self.platform.get_trace() def annotate_trace(self, key: str, value: str): self.platform.annotate_trace(key, value) def broadcast(self, route: str, headers: dict = None, body: any = None) -> None: self.util.validate_service_name(route) if headers is None and body is None: raise ValueError( 'Unable to broadcast because both headers and body are missing' ) event = EventEnvelope().set_to(route) if headers is not None: if not isinstance(headers, dict): raise ValueError('headers must be dict') for h in headers: event.set_header(h, str(headers[h])) if body is not None: event.set_body(body) self.platform.send_event(event, True) def send_later(self, route: str, headers: dict = None, body: any = None, seconds: float = 1.0) -> None: self.util.validate_service_name(route, True) if isinstance(seconds, float) or isinstance(seconds, int): relay = dict() relay['route'] = route if headers is not None: relay_headers = dict() for h in headers: relay_headers[str(h)] = str(headers[h]) relay['headers'] = relay_headers if body is not None: relay['body'] = body relay['seconds'] = seconds self.send(self.DEFERRED_DELIVERY, body=relay) else: raise ValueError('delay in seconds must be int or float') def send(self, route: str, headers: dict = None, body: any = None, reply_to: str = None, me=True) -> None: self.util.validate_service_name(route, True) if headers is None and body is None: raise ValueError( 'Unable to send because both headers and body are missing') event = EventEnvelope().set_to(route) if headers is not None: if not isinstance(headers, dict): raise ValueError('headers must be dict') for h in headers: event.set_header(str(h), str(headers[h])) if body is not None: event.set_body(body) if reply_to is not None: if not isinstance(reply_to, str): raise ValueError('reply_to must be str') # encode 'me' in the "call back" if replying to this instance event.set_reply_to(reply_to, me) self.platform.send_event(event) def request(self, route: str, timeout_seconds: float, headers: dict = None, body: any = None, correlation_id: str = None) -> EventEnvelope: self.util.validate_service_name(route, True) if headers is None and body is None: raise ValueError( 'Unable to make RPC call because both headers and body are missing' ) timeout_value = self.util.get_float(timeout_seconds) if timeout_value <= 0: raise ValueError( "timeout value in seconds must be positive number") event = EventEnvelope().set_to(route) if headers is not None: if not isinstance(headers, dict): raise ValueError('headers must be dict') for h in headers: event.set_header(h, str(headers[h])) if body is not None: event.set_body(body) if correlation_id is not None: event.set_correlation_id(str(correlation_id)) return self.platform.request(event, timeout_seconds) def parallel_request(self, events: list, timeout_seconds: float) -> list: return self.platform.parallel_request(events, timeout_seconds) def exists(self, routes: any): return self.platform.exists(routes)
def __init__(self): self.platform = Platform() self.util = Utility()
def hello(headers: dict, body: any, instance: int): # regular function signature (headers: dict, body: any, instance: int) Platform().get_logger().info("#" + str(instance) + " got ---> " + str(headers) + " body=" + str(body)) # as a demo, just echo the original payload return body