class ObjectStreamWriter:
def __init__(self, route: str):
if not isinstance(route, str):
raise ValueError('output stream-ID must be str')
self.closed = False
self.po = PostOffice()
self.output_stream = route
def write(self, payload: any):
if not self.closed:
if isinstance(payload, dict) or isinstance(payload, str) \
or isinstance(payload, bytes) \
or isinstance(payload, int) or isinstance(payload, float) or isinstance(payload, bool):
# for orderly write, use RPC request to guarantee that payload is written into the object stream
self.po.send(self.output_stream,
headers={'type': 'data'},
body=payload)
else:
raise ValueError(
'payload must be dict, str, bool, int or float')
def close(self):
if not self.closed:
self.closed = True
self.po.send(self.output_stream, headers={'type': 'eof'})
def main():
platform = Platform()
# you can register a method of a class
platform.register('hello.world.1', Hi().hello, 5)
# or register a function
platform.register('hello.world.2', hello, 10)
po = PostOffice()
# demonstrate sending asynchronously. Note that key-values in the headers will be encoded as strings
po.send('hello.world.1', headers={'one': 1}, body='hello world one')
po.send('hello.world.2', headers={'two': 2}, body='hello world two')
# demonstrate a RPC request
try:
result = po.request('hello.world.2', 2.0, headers={'some_key': 'some_value'}, body='hello world')
if isinstance(result, EventEnvelope):
print('Received RPC response:')
print("HEADERS =", result.get_headers(), ", BODY =", result.get_body(),
", STATUS =", result.get_status(),
", EXEC =", result.get_exec_time(), ", ROUND TRIP =", result.get_round_trip(), "ms")
except TimeoutError as e:
print("Exception: ", str(e))
# illustrate parallel RPC requests
event_list = list()
event_list.append(EventEnvelope().set_to('hello.world.1').set_body("first request"))
event_list.append(EventEnvelope().set_to('hello.world.2').set_body("second request"))
try:
result = po.parallel_request(event_list, 2.0)
if isinstance(result, list):
print('Received', len(result), 'RPC responses:')
for res in result:
print("HEADERS =", res.get_headers(), ", BODY =", res.get_body(),
", STATUS =", res.get_status(),
", EXEC =", res.get_exec_time(), ", ROUND TRIP =", res.get_round_trip(), "ms")
except TimeoutError as e:
print("Exception: ", str(e))
# connect to the network
platform.connect_to_cloud()
# wait until connected
while not platform.cloud_ready():
try:
time.sleep(0.1)
except KeyboardInterrupt:
# this allows us to stop the application while waiting for cloud connection
platform.stop()
return
# Demonstrate broadcast feature
# the event will be broadcast to multiple application instances that serve the same route
po.broadcast("hello.world.1", body="this is a broadcast message from "+platform.get_origin())
# demonstrate deferred delivery
po.send_later('hello.world.1', headers={'hello': 'world'}, body='this message arrives 5 seconds later', seconds=5.0)
#
# this will keep the main thread running in the background
# so we can use Control-C or KILL signal to stop the application
platform.run_forever()
def __init__(self, route: str):
if not isinstance(route, str):
raise ValueError('output stream-ID must be str')
self.closed = False
self.eof = False
self.po = PostOffice()
self.input_stream = route
self.stream = None
class ObjectStreamReader:
def __init__(self, route: str):
if not isinstance(route, str):
raise ValueError('output stream-ID must be str')
self.closed = False
self.eof = False
self.po = PostOffice()
self.input_stream = route
self.stream = None
def read(self, timeout_seconds: float):
if self.stream:
return self.stream()
if isinstance(timeout_seconds, int):
timeout_seconds = float(timeout_seconds)
if isinstance(timeout_seconds, float):
# minimum read timeout is one second
if timeout_seconds < 1.0:
timeout_seconds = 1.0
else:
raise ValueError('Read timeout must be float or int')
def reader():
while not self.eof:
# if input stream has nothing, it will throw TimeoutError
result = self.po.request(self.input_stream,
timeout_seconds,
headers={'type': 'read'})
if isinstance(result, EventEnvelope):
if result.get_status() == 200:
payload_type = result.get_headers().get('type')
if 'eof' == payload_type:
self.eof = True
yield None
if 'data' == payload_type:
yield result.get_body()
else:
raise AppException(result.get_status(),
str(result.get_body()))
self.stream = reader
return reader()
def close(self):
if not self.closed:
self.closed = True
self.po.request(self.input_stream, 10.0, headers={'type': 'close'})
class ObjectStreamIO:
STREAM_IO_MANAGER = 'object.streams.io'
def __init__(self, expiry_seconds: int = 1800):
self.po = PostOffice()
self.in_stream = None
self.out_stream = None
# create a new stream
if not isinstance(expiry_seconds, int):
raise ValueError('expiry_seconds must be int')
result = self.po.request(self.STREAM_IO_MANAGER,
6.0,
headers={
'type': 'create_stream',
'expiry': expiry_seconds
})
if isinstance(result, EventEnvelope) and isinstance(result.get_body(), dict) \
and result.get_status() == 200:
response: dict = result.get_body()
if 'in' in response and 'out' in response:
self.in_stream = response['in']
self.out_stream = response['out']
if self.in_stream is None:
raise IOError('Invalid response from stream manager')
def get_input_stream(self):
return self.in_stream
def get_output_stream(self):
return self.out_stream
def main():
platform = Platform()
# register a function
platform.register('hello.world', hello, 10)
po = PostOffice()
# demonstrate sending asynchronously. Note that key-values in the headers will be encoded as strings
po.send('hello.world', headers={'one': 1}, body='hello world one')
po.send('hello.world', headers={'two': 2}, body='hello world two')
# demonstrate a RPC request
try:
result = po.request('hello.world',
2.0,
headers={'some_key': 'some_value'},
body='hello world')
if isinstance(result, EventEnvelope):
print('Received RPC response:')
print("HEADERS =", result.get_headers(), ", BODY =",
result.get_body(), ", STATUS =", result.get_status(),
", EXEC =", result.get_exec_time(), ", ROUND TRIP =",
result.get_round_trip(), "ms")
except TimeoutError as e:
print("Exception: ", str(e))
# demonstrate drop-n-forget
for n in range(20):
po.send('hello.world', body='just a drop-n-forget message ' + str(n))
#
# this will keep the main thread running in the background
# so we can use Control-C or KILL signal to stop the application
platform.run_forever()
def __init__(self):
self.platform = Platform()
self.po = PostOffice()
self.util = Utility()
self.subscription = dict()
def subscription_sync(headers: dict, body: any):
if 'type' in headers and headers['type'] == 'subscription_sync':
if len(self.subscription) > 0:
for topic in self.subscription:
route_map = self.subscription[topic]
for route in route_map:
parameters = route_map[route]
self.platform.log.info('Update subscription ' +
topic + ' -> ' + route)
self.subscribe(topic, route, parameters)
else:
self.platform.log.info('No subscription to update')
self.platform.register('pub.sub.sync',
subscription_sync,
1,
is_private=True)
def __init__(self, route: str = None, expiry_seconds: int = 1800):
self.platform = Platform()
self.po = PostOffice()
self.util = Utility()
self.route = None
self.input_stream = None
self.output_stream = None
self.eof = False
self.input_closed = False
self.output_closed = False
if route is not None:
# open an existing stream
if isinstance(route, str):
name: str = route
if name.startswith('stream.') and '@' in name:
self.route = name
if self.route is None:
raise ValueError('Invalid stream route')
else:
# create a new stream
if not isinstance(expiry_seconds, int):
raise ValueError('expiry_seconds must be int')
result = self.po.request(self.STREAM_IO_MANAGER,
6.0,
headers={
'type': 'create',
'expiry_seconds': expiry_seconds
})
if isinstance(result, EventEnvelope) and isinstance(result.get_body(), str) \
and result.get_status() == 200:
name: str = result.get_body()
if name.startswith('stream.') and '@' in name:
self.route = name
if self.route is None:
raise IOError('Stream manager is not responding correctly')
class PubSub:
def __init__(self):
self.platform = Platform()
self.po = PostOffice()
self.util = Utility()
self.subscription = dict()
def subscription_sync(headers: dict, body: any):
if 'type' in headers and headers['type'] == 'subscription_sync':
if len(self.subscription) > 0:
for topic in self.subscription:
route_map = self.subscription[topic]
for route in route_map:
parameters = route_map[route]
self.platform.log.info('Update subscription ' +
topic + ' -> ' + route)
self.subscribe(topic, route, parameters)
else:
self.platform.log.info('No subscription to update')
self.platform.register('pub.sub.sync',
subscription_sync,
1,
is_private=True)
def feature_enabled(self):
result = self.po.request('pub.sub.controller',
10.0,
headers={'type': 'feature'})
return self._normalize_result(result, True)
def list_topics(self):
result = self.po.request('pub.sub.controller',
10.0,
headers={'type': 'list'})
return self._normalize_result(result, list())
def exists(self, topic: str):
if isinstance(topic, str):
result = self.po.request('pub.sub.controller',
10.0,
headers={
'type': 'exists',
'topic': topic
})
return self._normalize_result(result, True)
else:
return False
def create_topic(self, topic: str):
if isinstance(topic, str):
result = self.po.request('pub.sub.controller',
10.0,
headers={
'type': 'create',
'topic': topic
})
return self._normalize_result(result, True)
else:
raise ValueError("topic must be str")
def delete_topic(self, topic: str):
if isinstance(topic, str):
result = self.po.request('pub.sub.controller',
10.0,
headers={
'type': 'delete',
'topic': topic
})
return self._normalize_result(result, True)
else:
raise ValueError("topic must be str")
def publish(self, topic: str, headers: dict = None, body: any = None):
if isinstance(topic, str):
# encode payload
payload = dict()
payload['body'] = body
payload['headers'] = self._normalize_headers(headers)
result = self.po.request('pub.sub.controller',
10.0,
headers={
'type': 'publish',
'topic': topic
},
body=payload)
return self._normalize_result(result, True)
else:
raise ValueError("topic must be str")
def subscribe(self, topic: str, route: str, parameters: list = None):
if isinstance(topic, str) and isinstance(route, str):
if self.platform.has_route(route):
normalized_config = self._normalize_parameters(parameters)
result = self.po.request('pub.sub.controller',
10.0,
body=normalized_config,
headers={
'type': 'subscribe',
'topic': topic,
'route': route
})
done = self._normalize_result(result, True)
if done:
if topic not in self.subscription:
self.subscription[topic] = dict()
self.platform.log.info('Subscribed topic ' + topic)
route_map: dict = self.subscription[topic]
if route not in route_map:
route_map[route] = normalized_config
self.platform.log.info('Adding ' + route +
' to topic ' + topic)
return done
else:
raise ValueError("Unable to subscribe topic " + topic +
" because route " + route + " not registered")
else:
raise ValueError("topic and route must be str")
def unsubscribe(self, topic: str, route: str):
if isinstance(topic, str) and isinstance(route, str):
if self.platform.has_route(route):
result = self.po.request('pub.sub.controller',
10.0,
headers={
'type': 'unsubscribe',
'topic': topic,
'route': route
})
done = self._normalize_result(result, True)
if done:
if topic in self.subscription:
route_map: dict = self.subscription[topic]
if route in route_map:
route_map.pop(route)
self.platform.log.info('Removing ' + route +
' from topic ' + topic)
if len(route_map) == 0:
self.subscription.pop(topic)
self.platform.log.info('Unsubscribed topic ' +
topic)
return done
else:
raise ValueError("Unable to unsubscribe topic " + topic +
" because route " + route + " not registered")
else:
raise ValueError("topic and route must be str")
@staticmethod
def _normalize_result(result: EventEnvelope, result_obj: any):
if isinstance(result, EventEnvelope):
if result.get_status() == 200:
if isinstance(result.get_body(), type(result_obj)):
return result.get_body()
else:
raise AppException(500, str(result.get_body()))
else:
raise AppException(result.get_status(), str(result.get_body()))
@staticmethod
def _normalize_headers(headers: dict):
if headers is None:
return dict()
if isinstance(headers, dict):
result = dict()
for h in headers:
result[str(h)] = str(headers[h])
return result
else:
raise ValueError("headers must be dict of str key-values")
@staticmethod
def _normalize_parameters(parameters: list):
if parameters is None:
return list()
if isinstance(parameters, list):
result = list()
for h in parameters:
result.append(str(h))
return result
else:
raise ValueError("headers must be a list of str")
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from mercury.platform import Platform
from mercury.system.models import EventEnvelope
from mercury.system.po import PostOffice
platform = Platform()
log = platform.get_logger()
po = PostOffice()
def hello(headers: dict, body: any, instance: int):
# regular function signature (headers: dict, body: any, instance: int)
log.info(f'#{instance} got header={headers} body={body}')
# as a demo, just echo the original payload
return body
def main():
# register a function
platform.register('hello.world', hello, 10)
# demonstrate sending asynchronously. Note that key-values in the headers will be encoded as strings
po.send('hello.world', headers={'one': 1}, body='hello world one')
class ObjectStreamIO:
STREAM_IO_MANAGER = 'object.streams.io'
def __init__(self, route: str = None, expiry_seconds: int = 1800):
self.platform = Platform()
self.po = PostOffice()
self.util = Utility()
self.route = None
self.input_stream = None
self.output_stream = None
self.eof = False
self.input_closed = False
self.output_closed = False
if route is not None:
# open an existing stream
if isinstance(route, str):
name: str = route
if name.startswith('stream.') and '@' in name:
self.route = name
if self.route is None:
raise ValueError('Invalid stream route')
else:
# create a new stream
if not isinstance(expiry_seconds, int):
raise ValueError('expiry_seconds must be int')
result = self.po.request(self.STREAM_IO_MANAGER,
6.0,
headers={
'type': 'create',
'expiry_seconds': expiry_seconds
})
if isinstance(result, EventEnvelope) and isinstance(result.get_body(), str) \
and result.get_status() == 200:
name: str = result.get_body()
if name.startswith('stream.') and '@' in name:
self.route = name
if self.route is None:
raise IOError('Stream manager is not responding correctly')
def get_route(self):
return self.route
def is_eof(self):
return self.eof
def read(self, timeout_seconds: float):
if self.input_stream:
return self.input_stream()
if isinstance(timeout_seconds, int):
timeout_seconds = float(timeout_seconds)
if isinstance(timeout_seconds, float):
# minimum read timeout is one second
if timeout_seconds < 1.0:
timeout_seconds = 1.0
else:
raise ValueError('Read timeout must be float or int')
def reader():
while not self.eof:
# if input stream has nothing, it will throw TimeoutError
result = self.po.request(self.route,
timeout_seconds,
headers={'type': 'read'})
if isinstance(result, EventEnvelope):
if result.get_status() == 200:
payload_type = result.get_headers().get('type')
if 'eof' == payload_type:
self.eof = True
break
if 'body' == payload_type:
yield result.get_body()
else:
raise AppException(result.get_status(),
str(result.get_body()))
self.input_stream = reader
return reader()
def write(self, payload: any, timeout_seconds: float = 10.0):
if isinstance(timeout_seconds, int):
timeout_seconds = float(timeout_seconds)
if isinstance(timeout_seconds, float):
# minimum write timeout is five second
if timeout_seconds < 5.0:
timeout_seconds = 5.0
else:
raise ValueError('Write timeout must be float or int')
if not self.output_closed:
if isinstance(payload, dict) or isinstance(payload, str) or isinstance(payload, bool) \
or isinstance(payload, int) or isinstance(payload, float):
# for orderly write, use RPC request to guarantee that payload is written into the object stream
self.po.request(self.route,
timeout_seconds,
headers={'type': 'write'},
body=payload)
else:
raise ValueError(
'payload must be dict, str, bool, int or float')
def send_eof(self):
if not self.output_closed:
self.output_closed = True
self.po.send(self.route, headers={'type': 'eof'})
def is_output_closed(self):
return self.output_closed
def get_local_streams(self):
result = self.po.request(self.STREAM_IO_MANAGER,
6.0,
headers={'type': 'query'})
if isinstance(result, EventEnvelope) and isinstance(result.get_body(), dict) \
and result.get_status() == 200:
return result.get_body()
else:
return dict()
def close(self):
if not self.input_closed:
self.input_closed = True
self.po.send(self.route, headers={'type': 'close'})
def is_input_closed(self):
return self.input_closed