def main():
platform = Platform()
platform.connect_to_cloud()
# wait until connected
while not platform.cloud_ready():
try:
time.sleep(0.1)
except KeyboardInterrupt:
# this allows us to stop the application while waiting for cloud connection
platform.stop()
return
util = Utility()
pubsub = PubSub()
if pubsub.feature_enabled():
# Publish an event
# headers = optional parameters for the event
# body = event payload
for x in range(10):
print("publishing event#", x)
pubsub.publish("hello.topic",
headers={
"some_parameter": "some_value",
"n": x
},
body="hello world " +
util.get_iso_8601(time.time()))
# quit application
platform.stop()
else:
print("Pub/Sub feature not available from the underlying event stream")
print("Did you start the language connector with Kafka?")
print(
"e.g. java -Dcloud.connector=kafka -Dcloud.services=kafka.reporter -jar language-connector-1.12.31.jar"
)
def main():
platform = Platform()
log = platform.get_logger()
# connect to the network
platform.connect_to_cloud()
# wait until connected
while not platform.cloud_ready():
try:
time.sleep(0.1)
except KeyboardInterrupt:
# this allows us to stop the application while waiting for cloud connection
platform.stop()
return
#
# You can create a new I/O stream using ObjectStreamIO.
# This requires a live connection to the language connector.
#
stream = ObjectStreamIO(10)
in_stream_id = stream.get_input_stream()
out_stream_id = stream.get_output_stream()
output_stream = ObjectStreamWriter(out_stream_id)
input_stream = ObjectStreamReader(in_stream_id)
for i in range(100):
output_stream.write('hello world ' + str(i))
#
# if output stream is not closed, input will timeout
# Therefore, please use try-except for TimeoutError in the iterator for-loop below.
#
output_stream.close()
for block in input_stream.read(5.0):
if block is None:
log.info("EOF")
else:
log.info(block)
input_stream.close()
#
# this will keep the main thread running in the background
# so we can use Control-C or KILL signal to stop the application
platform.run_forever()
def hello(headers: dict, body: any, instance: int):
# regular function signature (headers: dict, body: any, instance: int)
Platform().log.info("#"+str(instance)+" "+str(headers)+" body="+str(body))
# to set status, headers and body, return them in an event envelope
result = EventEnvelope().set_header('hello', 'world').set_body(body)
for h in headers:
result.set_header(h, headers[h])
return result
def main():
platform = Platform()
# register a function
platform.register('hello.world', hello, 10)
po = PostOffice()
# demonstrate sending asynchronously. Note that key-values in the headers will be encoded as strings
po.send('hello.world', headers={'one': 1}, body='hello world one')
po.send('hello.world', headers={'two': 2}, body='hello world two')
# demonstrate a RPC request
try:
result = po.request('hello.world',
2.0,
headers={'some_key': 'some_value'},
body='hello world')
if isinstance(result, EventEnvelope):
print('Received RPC response:')
print("HEADERS =", result.get_headers(), ", BODY =",
result.get_body(), ", STATUS =", result.get_status(),
", EXEC =", result.get_exec_time(), ", ROUND TRIP =",
result.get_round_trip(), "ms")
except TimeoutError as e:
print("Exception: ", str(e))
# demonstrate drop-n-forget
for n in range(20):
po.send('hello.world', body='just a drop-n-forget message ' + str(n))
#
# this will keep the main thread running in the background
# so we can use Control-C or KILL signal to stop the application
platform.run_forever()
def __init__(self):
self.platform = Platform()
self.po = PostOffice()
self.util = Utility()
self.subscription = dict()
def subscription_sync(headers: dict, body: any):
if 'type' in headers and headers['type'] == 'subscription_sync':
if len(self.subscription) > 0:
for topic in self.subscription:
route_map = self.subscription[topic]
for route in route_map:
parameters = route_map[route]
self.platform.log.info('Update subscription ' +
topic + ' -> ' + route)
self.subscribe(topic, route, parameters)
else:
self.platform.log.info('No subscription to update')
self.platform.register('pub.sub.sync',
subscription_sync,
1,
is_private=True)
def __init__(self, route: str = None, expiry_seconds: int = 1800):
self.platform = Platform()
self.po = PostOffice()
self.util = Utility()
self.route = None
self.input_stream = None
self.output_stream = None
self.eof = False
self.input_closed = False
self.output_closed = False
if route is not None:
# open an existing stream
if isinstance(route, str):
name: str = route
if name.startswith('stream.') and '@' in name:
self.route = name
if self.route is None:
raise ValueError('Invalid stream route')
else:
# create a new stream
if not isinstance(expiry_seconds, int):
raise ValueError('expiry_seconds must be int')
result = self.po.request(self.STREAM_IO_MANAGER,
6.0,
headers={
'type': 'create',
'expiry_seconds': expiry_seconds
})
if isinstance(result, EventEnvelope) and isinstance(result.get_body(), str) \
and result.get_status() == 200:
name: str = result.get_body()
if name.startswith('stream.') and '@' in name:
self.route = name
if self.route is None:
raise IOError('Stream manager is not responding correctly')
class PostOffice:
"""
Convenient class for making RPC, async and callback.
"""
def __init__(self):
self.platform = Platform()
self.util = Utility()
def get_route(self):
"""
Obtain my route name for the currently running service.
This is useful for Role Based Access Control (RBAC) to restrict certain user roles for a service.
Note that RBAC is the responsibility of the user application.
Returns: route name
"""
trace_info = self.get_trace()
return "?" if trace_info is None else trace_info.get_route()
def get_trace_id(self):
return self.platform.get_trace_id()
def get_trace(self):
return self.platform.get_trace()
def annotate_trace(self, key: str, value: str):
self.platform.annotate_trace(key, value)
def broadcast(self,
route: str,
headers: dict = None,
body: any = None) -> None:
self.util.validate_service_name(route)
if headers is None and body is None:
raise ValueError(
'Unable to broadcast because both headers and body are missing'
)
event = EventEnvelope().set_to(route)
if headers is not None:
if not isinstance(headers, dict):
raise ValueError('headers must be dict')
for h in headers:
event.set_header(h, str(headers[h]))
if body is not None:
event.set_body(body)
self.platform.send_event(event, True)
def send_later(self,
route: str,
headers: dict = None,
body: any = None,
reply_to: str = None,
me=True,
seconds: float = 1.0) -> None:
self.util.validate_service_name(route, True)
if isinstance(seconds, float) or isinstance(seconds, int):
if seconds > 0:
if headers is None and body is None:
raise ValueError(
'Unable to send because both headers and body are missing'
)
event = EventEnvelope().set_to(route)
if headers is not None:
if not isinstance(headers, dict):
raise ValueError('headers must be dict')
for h in headers:
event.set_header(str(h), str(headers[h]))
if body is not None:
event.set_body(body)
if reply_to is not None:
if not isinstance(reply_to, str):
raise ValueError('reply_to must be str')
# encode 'me' in the "call back" if replying to this instance
event.set_reply_to(reply_to, me)
self.platform.send_event_later(event, seconds)
else:
raise ValueError('delay in seconds must be larger than zero')
else:
raise ValueError('delay in seconds must be int or float')
def send_event_later(self,
event: EventEnvelope,
seconds: float = 1.0,
me=True):
if event.get_reply_to() is not None:
event.set_reply_to(event.get_reply_to(), me)
self.platform.send_event_later(event, seconds)
def send(self,
route: str,
headers: dict = None,
body: any = None,
reply_to: str = None,
me=True) -> None:
self.util.validate_service_name(route, True)
if headers is None and body is None:
raise ValueError(
'Unable to send because both headers and body are missing')
event = EventEnvelope().set_to(route)
if headers is not None:
if not isinstance(headers, dict):
raise ValueError('headers must be dict')
for h in headers:
event.set_header(str(h), str(headers[h]))
if body is not None:
event.set_body(body)
if reply_to is not None:
if not isinstance(reply_to, str):
raise ValueError('reply_to must be str')
# encode 'me' in the "call back" if replying to this instance
event.set_reply_to(reply_to, me)
self.platform.send_event(event)
def send_event(self, event: EventEnvelope, me=True):
if event.get_reply_to() is not None:
event.set_reply_to(event.get_reply_to(), me)
self.platform.send_event(event)
def request(self,
route: str,
timeout_seconds: float,
headers: dict = None,
body: any = None,
correlation_id: str = None) -> EventEnvelope:
self.util.validate_service_name(route, True)
if headers is None and body is None:
raise ValueError(
'Unable to make RPC call because both headers and body are missing'
)
timeout_value = self.util.get_float(timeout_seconds)
if timeout_value <= 0:
raise ValueError(
'timeout value in seconds must be positive number')
event = EventEnvelope().set_to(route)
if headers is not None:
if not isinstance(headers, dict):
raise ValueError('headers must be dict')
for h in headers:
event.set_header(h, str(headers[h]))
if body is not None:
event.set_body(body)
if correlation_id is not None:
event.set_correlation_id(str(correlation_id))
response = self.platform.request(event, timeout_seconds)
if isinstance(response, EventEnvelope):
if response.get_tag('exception') is None:
return response
else:
raise AppException(response.get_status(), response.get_body())
raise ValueError(
f'Expect response is EventEnvelope, actual: ({response})')
def single_request(self, event: EventEnvelope, timeout_seconds: float):
response = self.platform.request(event, timeout_seconds)
if isinstance(response, EventEnvelope):
if response.get_tag('exception') is None:
return response
else:
raise AppException(response.get_status(), response.get_body())
raise ValueError(
f'Expect response is EventEnvelope, actual: ({response})')
def parallel_request(self, events: list, timeout_seconds: float) -> list:
return self.platform.parallel_request(events, timeout_seconds)
def exists(self, routes: any):
return self.platform.exists(routes)
def main():
platform = Platform()
# you can register a method of a class
platform.register('hello.world.1', Hi().hello, 5)
# or register a function
platform.register('hello.world.2', hello, 10)
po = PostOffice()
# demonstrate sending asynchronously. Note that key-values in the headers will be encoded as strings
po.send('hello.world.1', headers={'one': 1}, body='hello world one')
po.send('hello.world.2', headers={'two': 2}, body='hello world two')
# demonstrate a RPC request
try:
result = po.request('hello.world.2',
2.0,
headers={'some_key': 'some_value'},
body='hello world')
if isinstance(result, EventEnvelope):
print('Received RPC response:')
print("HEADERS =", result.get_headers(), ", BODY =",
result.get_body(), ", STATUS =", result.get_status(),
", EXEC =", result.get_exec_time(), ", ROUND TRIP =",
result.get_round_trip(), "ms")
except TimeoutError as e:
print("Exception: ", str(e))
# illustrate parallel RPC requests
event_list = list()
event_list.append(
EventEnvelope().set_to('hello.world.1').set_body("first request"))
event_list.append(
EventEnvelope().set_to('hello.world.2').set_body("second request"))
try:
result = po.parallel_request(event_list, 2.0)
if isinstance(result, list):
print('Received', len(result), 'RPC responses:')
for res in result:
print("HEADERS =", res.get_headers(), ", BODY =",
res.get_body(), ", STATUS =", res.get_status(),
", EXEC =", res.get_exec_time(), ", ROUND TRIP =",
res.get_round_trip(), "ms")
except TimeoutError as e:
print("Exception: ", str(e))
# connect to the network
platform.connect_to_cloud()
# wait until connected
while not platform.cloud_ready():
try:
time.sleep(0.1)
except KeyboardInterrupt:
# this allows us to stop the application while waiting for cloud connection
platform.stop()
return
# Demonstrate broadcast feature
# the event will be broadcast to multiple application instances that serve the same route
po.broadcast("hello.world.1",
body="this is a broadcast message from " +
platform.get_origin())
# demonstrate deferred delivery
po.send_later('hello.world.1',
headers={'hello': 'world'},
body='this message arrives 5 seconds later',
seconds=5.0)
#
# this will keep the main thread running in the background
# so we can use Control-C or KILL signal to stop the application
platform.run_forever()
def hello(self, headers: dict, body: any):
# singleton function signature (headers: dict, body: any)
Platform().get_logger().info(self.MY_NAME + " " + str(headers) + ", " +
str(body))
return body
def main():
platform = Platform()
# this shows that we can register a route name for a function
platform.register('hello.world', hello, 10)
# Once it connects to the network, it is ready to serve requests
platform.connect_to_cloud()
# wait until connected
while not platform.cloud_ready():
try:
time.sleep(0.1)
except KeyboardInterrupt:
# this allows us to stop the application while waiting for cloud connection
platform.stop()
return
#
# this will keep the main thread running in the background
# so we can use Control-C or KILL signal to stop the application
platform.run_forever()
class PubSub:
def __init__(self):
self.platform = Platform()
self.po = PostOffice()
self.util = Utility()
self.subscription = dict()
def subscription_sync(headers: dict, body: any):
if 'type' in headers and headers['type'] == 'subscription_sync':
if len(self.subscription) > 0:
for topic in self.subscription:
route_map = self.subscription[topic]
for route in route_map:
parameters = route_map[route]
self.platform.log.info('Update subscription ' +
topic + ' -> ' + route)
self.subscribe(topic, route, parameters)
else:
self.platform.log.info('No subscription to update')
self.platform.register('pub.sub.sync',
subscription_sync,
1,
is_private=True)
def feature_enabled(self):
result = self.po.request('pub.sub.controller',
10.0,
headers={'type': 'feature'})
return self._normalize_result(result, True)
def list_topics(self):
result = self.po.request('pub.sub.controller',
10.0,
headers={'type': 'list'})
return self._normalize_result(result, list())
def exists(self, topic: str):
if isinstance(topic, str):
result = self.po.request('pub.sub.controller',
10.0,
headers={
'type': 'exists',
'topic': topic
})
return self._normalize_result(result, True)
else:
return False
def create_topic(self, topic: str):
if isinstance(topic, str):
result = self.po.request('pub.sub.controller',
10.0,
headers={
'type': 'create',
'topic': topic
})
return self._normalize_result(result, True)
else:
raise ValueError("topic must be str")
def delete_topic(self, topic: str):
if isinstance(topic, str):
result = self.po.request('pub.sub.controller',
10.0,
headers={
'type': 'delete',
'topic': topic
})
return self._normalize_result(result, True)
else:
raise ValueError("topic must be str")
def publish(self, topic: str, headers: dict = None, body: any = None):
if isinstance(topic, str):
# encode payload
payload = dict()
payload['body'] = body
payload['headers'] = self._normalize_headers(headers)
result = self.po.request('pub.sub.controller',
10.0,
headers={
'type': 'publish',
'topic': topic
},
body=payload)
return self._normalize_result(result, True)
else:
raise ValueError("topic must be str")
def subscribe(self, topic: str, route: str, parameters: list = None):
if isinstance(topic, str) and isinstance(route, str):
if self.platform.has_route(route):
normalized_config = self._normalize_parameters(parameters)
result = self.po.request('pub.sub.controller',
10.0,
body=normalized_config,
headers={
'type': 'subscribe',
'topic': topic,
'route': route
})
done = self._normalize_result(result, True)
if done:
if topic not in self.subscription:
self.subscription[topic] = dict()
self.platform.log.info('Subscribed topic ' + topic)
route_map: dict = self.subscription[topic]
if route not in route_map:
route_map[route] = normalized_config
self.platform.log.info('Adding ' + route +
' to topic ' + topic)
return done
else:
raise ValueError("Unable to subscribe topic " + topic +
" because route " + route + " not registered")
else:
raise ValueError("topic and route must be str")
def unsubscribe(self, topic: str, route: str):
if isinstance(topic, str) and isinstance(route, str):
if self.platform.has_route(route):
result = self.po.request('pub.sub.controller',
10.0,
headers={
'type': 'unsubscribe',
'topic': topic,
'route': route
})
done = self._normalize_result(result, True)
if done:
if topic in self.subscription:
route_map: dict = self.subscription[topic]
if route in route_map:
route_map.pop(route)
self.platform.log.info('Removing ' + route +
' from topic ' + topic)
if len(route_map) == 0:
self.subscription.pop(topic)
self.platform.log.info('Unsubscribed topic ' +
topic)
return done
else:
raise ValueError("Unable to unsubscribe topic " + topic +
" because route " + route + " not registered")
else:
raise ValueError("topic and route must be str")
@staticmethod
def _normalize_result(result: EventEnvelope, result_obj: any):
if isinstance(result, EventEnvelope):
if result.get_status() == 200:
if isinstance(result.get_body(), type(result_obj)):
return result.get_body()
else:
raise AppException(500, str(result.get_body()))
else:
raise AppException(result.get_status(), str(result.get_body()))
@staticmethod
def _normalize_headers(headers: dict):
if headers is None:
return dict()
if isinstance(headers, dict):
result = dict()
for h in headers:
result[str(h)] = str(headers[h])
return result
else:
raise ValueError("headers must be dict of str key-values")
@staticmethod
def _normalize_parameters(parameters: list):
if parameters is None:
return list()
if isinstance(parameters, list):
result = list()
for h in parameters:
result.append(str(h))
return result
else:
raise ValueError("headers must be a list of str")
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from mercury.platform import Platform
from mercury.system.models import EventEnvelope
from mercury.system.po import PostOffice
platform = Platform()
log = platform.get_logger()
po = PostOffice()
def hello(headers: dict, body: any, instance: int):
# regular function signature (headers: dict, body: any, instance: int)
log.info(f'#{instance} got header={headers} body={body}')
# as a demo, just echo the original payload
return body
def main():
# register a function
platform.register('hello.world', hello, 10)
def main():
platform = Platform()
# we should register the custom trace processor as a singleton
platform.register('distributed.trace.processor', my_trace_processor, 1)
# Once it connects to the network, it is ready to serve requests
platform.connect_to_cloud()
# wait until connected
while not platform.cloud_ready():
try:
time.sleep(0.1)
except KeyboardInterrupt:
# this allows us to stop the application while waiting for cloud connection
platform.stop()
return
#
# this will keep the main thread running in the background
# so we can use Control-C or KILL signal to stop the application
platform.run_forever()
def main():
platform = Platform()
# register a route name for a pub/sub subscriber function
# setting number of instance to 1 because pub/sub subscriber is always a singleton
platform.register('hello.world', hello, 1)
# Once it connects to the network, it is ready to serve requests
platform.connect_to_cloud()
# wait until connected
while not platform.cloud_ready():
try:
time.sleep(0.1)
except KeyboardInterrupt:
# this allows us to stop the application while waiting for cloud connection
platform.stop()
return
pubsub = PubSub()
if pubsub.feature_enabled():
#
# the pub/sub topic name must be different from the subscriber function name
#
# Note:
# For kafka, the parameter list includes the following:
# client_id, group_id and optional offset number (as a string)
# e.g. ["client1", "group1"] or ["client1", "group1", "0"]
#
# In this example, it is reading from the beginning of the topic.
# For a real application, it should read without the offset so that it can fetch the latest events.
#
pubsub.subscribe("hello.topic", "hello.world",
["client1", "group1", "0"])
else:
print("Pub/Sub feature not available from the underlying event stream")
print("Did you start the language connector with Kafka?")
print(
"e.g. java -Dcloud.connector=kafka -Dcloud.services=kafka.reporter -jar language-connector-1.12.31.jar"
)
#
# this will keep the main thread running in the background
# so we can use Control-C or KILL signal to stop the application
platform.run_forever()
class PostOffice:
"""
Convenient class for making RPC, async and callback.
"""
DEFERRED_DELIVERY = 'system.deferred.delivery'
def __init__(self):
self.platform = Platform()
self.util = Utility()
def get_route(self):
"""
Obtain my route name for the currently running service.
This is useful for Role Based Access Control (RBAC) to restrict certain user roles for a service.
Note that RBAC is the responsibility of the user application.
:return: route name
"""
trace_info = self.get_trace()
return "?" if trace_info is None else trace_info.get_route()
def get_trace_id(self):
return self.platform.get_trace_id()
def get_trace(self):
return self.platform.get_trace()
def annotate_trace(self, key: str, value: str):
self.platform.annotate_trace(key, value)
def broadcast(self,
route: str,
headers: dict = None,
body: any = None) -> None:
self.util.validate_service_name(route)
if headers is None and body is None:
raise ValueError(
'Unable to broadcast because both headers and body are missing'
)
event = EventEnvelope().set_to(route)
if headers is not None:
if not isinstance(headers, dict):
raise ValueError('headers must be dict')
for h in headers:
event.set_header(h, str(headers[h]))
if body is not None:
event.set_body(body)
self.platform.send_event(event, True)
def send_later(self,
route: str,
headers: dict = None,
body: any = None,
seconds: float = 1.0) -> None:
self.util.validate_service_name(route, True)
if isinstance(seconds, float) or isinstance(seconds, int):
relay = dict()
relay['route'] = route
if headers is not None:
relay_headers = dict()
for h in headers:
relay_headers[str(h)] = str(headers[h])
relay['headers'] = relay_headers
if body is not None:
relay['body'] = body
relay['seconds'] = seconds
self.send(self.DEFERRED_DELIVERY, body=relay)
else:
raise ValueError('delay in seconds must be int or float')
def send(self,
route: str,
headers: dict = None,
body: any = None,
reply_to: str = None,
me=True) -> None:
self.util.validate_service_name(route, True)
if headers is None and body is None:
raise ValueError(
'Unable to send because both headers and body are missing')
event = EventEnvelope().set_to(route)
if headers is not None:
if not isinstance(headers, dict):
raise ValueError('headers must be dict')
for h in headers:
event.set_header(str(h), str(headers[h]))
if body is not None:
event.set_body(body)
if reply_to is not None:
if not isinstance(reply_to, str):
raise ValueError('reply_to must be str')
# encode 'me' in the "call back" if replying to this instance
event.set_reply_to(reply_to, me)
self.platform.send_event(event)
def request(self,
route: str,
timeout_seconds: float,
headers: dict = None,
body: any = None,
correlation_id: str = None) -> EventEnvelope:
self.util.validate_service_name(route, True)
if headers is None and body is None:
raise ValueError(
'Unable to make RPC call because both headers and body are missing'
)
timeout_value = self.util.get_float(timeout_seconds)
if timeout_value <= 0:
raise ValueError(
"timeout value in seconds must be positive number")
event = EventEnvelope().set_to(route)
if headers is not None:
if not isinstance(headers, dict):
raise ValueError('headers must be dict')
for h in headers:
event.set_header(h, str(headers[h]))
if body is not None:
event.set_body(body)
if correlation_id is not None:
event.set_correlation_id(str(correlation_id))
return self.platform.request(event, timeout_seconds)
def parallel_request(self, events: list, timeout_seconds: float) -> list:
return self.platform.parallel_request(events, timeout_seconds)
def exists(self, routes: any):
return self.platform.exists(routes)
def __init__(self):
self.platform = Platform()
self.util = Utility()
def hello(headers: dict, body: any, instance: int):
# regular function signature (headers: dict, body: any, instance: int)
Platform().get_logger().info("#" + str(instance) + " got ---> " +
str(headers) + " body=" + str(body))
# as a demo, just echo the original payload
return body
