def run(loop):
nc = NATS()
yield from nc.connect("127.0.0.1:4222", loop=loop)
@asyncio.coroutine
def message_handler(msg):
subject = msg.subject
reply = msg.reply
data = msg.data.decode()
print("Received a message on '{subject} {reply}': {data}".format(
subject=subject, reply=reply, data=data))
# Simple publisher and async subscriber via coroutine.
sid = yield from nc.subscribe("foo", cb=message_handler)
# Stop receiving after 2 messages.
yield from nc.auto_unsubscribe(sid, 2)
yield from nc.publish("foo", b'Hello')
yield from nc.publish("foo", b'World')
yield from nc.publish("foo", b'!!!!!')
@asyncio.coroutine
def help_request(msg):
subject = msg.subject
reply = msg.reply
data = msg.data.decode()
print("Received a message on '{subject} {reply}': {data}".format(
subject=subject, reply=reply, data=data))
yield from nc.publish(reply, b'I can help')
# Use queue named 'workers' for distributing requests
# among subscribers.
sid = yield from nc.subscribe("help", "workers", help_request)
# Send a request and expect a single response
# and trigger timeout if not faster than 50 ms.
try:
response = yield from nc.request("help", b'help me', 0.050)
print("Received response: {message}".format(
message=response.data.decode()))
except ErrTimeout:
print("Request timed out")
# Remove interest in subscription.
yield from nc.unsubscribe(sid)
yield from nc.close()
def test_unsubscribe(self):
nc = NATS()
msgs = []
@asyncio.coroutine
def subscription_handler(msg):
msgs.append(msg)
yield from nc.connect(io_loop=self.loop)
sid = yield from nc.subscribe("foo", cb=subscription_handler)
yield from nc.publish("foo", b'A')
yield from nc.publish("foo", b'B')
# Wait a bit to receive the messages
yield from asyncio.sleep(0.5, loop=self.loop)
self.assertEqual(2, len(msgs))
yield from nc.unsubscribe(sid)
yield from nc.publish("foo", b'C')
yield from nc.publish("foo", b'D')
# Ordering should be preserverd in these at least
self.assertEqual(b'A', msgs[0].data)
self.assertEqual(b'B', msgs[1].data)
# Should not exist by now
with self.assertRaises(KeyError):
nc._subs[sid].received
yield from asyncio.sleep(1, loop=self.loop)
endpoint = '127.0.0.1:{port}'.format(
port=self.server_pool[0].http_port)
httpclient = http.client.HTTPConnection(endpoint, timeout=5)
httpclient.request('GET', '/connz')
response = httpclient.getresponse()
connz = json.loads((response.read()).decode())
self.assertEqual(1, len(connz['connections']))
self.assertEqual(0, connz['connections'][0]['subscriptions'])
self.assertEqual(4, connz['connections'][0]['in_msgs'])
self.assertEqual(4, connz['connections'][0]['in_bytes'])
self.assertEqual(2, connz['connections'][0]['out_msgs'])
self.assertEqual(2, connz['connections'][0]['out_bytes'])
yield from nc.close()
self.assertEqual(2, nc.stats['in_msgs'])
self.assertEqual(2, nc.stats['in_bytes'])
self.assertEqual(4, nc.stats['out_msgs'])
self.assertEqual(4, nc.stats['out_bytes'])
def test_unsubscribe(self):
nc = NATS()
msgs = []
@asyncio.coroutine
def subscription_handler(msg):
msgs.append(msg)
yield from nc.connect(io_loop=self.loop)
sid = yield from nc.subscribe("foo", cb=subscription_handler)
yield from nc.publish("foo", b'A')
yield from nc.publish("foo", b'B')
# Wait a bit to receive the messages
yield from asyncio.sleep(0.5, loop=self.loop)
self.assertEqual(2, len(msgs))
yield from nc.unsubscribe(sid)
yield from nc.publish("foo", b'C')
yield from nc.publish("foo", b'D')
# Ordering should be preserverd in these at least
self.assertEqual(b'A', msgs[0].data)
self.assertEqual(b'B', msgs[1].data)
# Should not exist by now
with self.assertRaises(KeyError):
nc._subs[sid].received
yield from asyncio.sleep(1, loop=self.loop)
endpoint = '127.0.0.1:{port}'.format(
port=self.server_pool[0].http_port)
httpclient = http.client.HTTPConnection(endpoint, timeout=5)
httpclient.request('GET', '/connz')
response = httpclient.getresponse()
connz = json.loads((response.read()).decode())
self.assertEqual(1, len(connz['connections']))
self.assertEqual(0, connz['connections'][0]['subscriptions'])
self.assertEqual(4, connz['connections'][0]['in_msgs'])
self.assertEqual(4, connz['connections'][0]['in_bytes'])
self.assertEqual(2, connz['connections'][0]['out_msgs'])
self.assertEqual(2, connz['connections'][0]['out_bytes'])
yield from nc.close()
self.assertEqual(2, nc.stats['in_msgs'])
self.assertEqual(2, nc.stats['in_bytes'])
self.assertEqual(4, nc.stats['out_msgs'])
self.assertEqual(4, nc.stats['out_bytes'])
class Service(pykka.ThreadingActor):
"""
NATS service implementation.
:param my_args: dict like {connection, service_q, treatment_callback[, service_name]}
:param connection_args: dict like {user, password, host[, port, client_properties]}
"""
def __init__(self, my_args=None, connection_args=None):
"""
NATS service constructor
:param my_args: dict like {connection, service_q, treatment_callback[, service_name]}
:param connection_args: dict like {user, password, host[, port, client_properties]}
:return: self
"""
LOGGER.debug("natsd.Service.__init__")
if my_args is None or connection_args is None:
raise exceptions.ArianeConfError("service arguments")
if 'service_q' not in my_args or my_args['service_q'] is None or not my_args['service_q']:
raise exceptions.ArianeConfError("service_q")
if 'treatment_callback' not in my_args or my_args['treatment_callback'] is None:
raise exceptions.ArianeConfError("treatment_callback")
if 'service_name' not in my_args or my_args['service_name'] is None or not my_args['service_name']:
LOGGER.warn("natsd.Service.__init__ - service_name is not defined ! Use default : " +
self.__class__.__name__)
my_args['service_name'] = self.__class__.__name__
Driver.validate_driver_conf(connection_args)
super(Service, self).__init__()
self.connection_args = copy.deepcopy(connection_args)
self.servers = [
"nats://" + connection_args['user'] + ":" + connection_args['password'] + "@" +
connection_args['host']+":"+str(connection_args['port'])
]
self.name = self.connection_args['client_properties']['ariane.app'] + "@" + socket.gethostname() + \
" - service on " + my_args['service_q']
self.loop = None
self.options = None
self.nc = Client()
self.serviceQ = my_args['service_q']
self.serviceQS = None
self.service_name = my_args['service_name']
self.service = None
self.cb = my_args['treatment_callback']
self.is_started = False
def on_request(self, msg):
"""
message consumed treatment through provided callback and basic ack
"""
LOGGER.debug("natsd.Service.on_request - request " + str(msg) + " received")
try:
working_response = json.loads(msg.data.decode())
working_properties = DriverTools.json2properties(working_response['properties'])
working_body = b''+bytes(working_response['body'], 'utf8') if 'body' in working_response else None
working_body_decoded = base64.b64decode(working_body) if working_body is not None else \
bytes(json.dumps({}), 'utf8')
self.cb(working_properties, working_body_decoded)
except Exception as e:
LOGGER.warn("natsd.Service.on_request - Exception raised while treating msg {"+str(msg)+","+str(msg)+"}")
LOGGER.debug("natsd.Service.on_request - request " + str(msg) + " treated")
def connect(self):
LOGGER.debug("natsd.Service.connect")
try:
yield from self.nc.connect(**self.options)
self.serviceQS = yield from self.nc.subscribe(self.serviceQ, cb=self.on_request)
self.is_started = True
except ErrNoServers as e:
print(e)
return
def run_event_loop(self):
LOGGER.debug("natsd.Service.run_event_loop")
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
self.options = {
"servers": self.servers,
"name": self.name,
# "disconnected_cb": self.disconnected_cb,
# "reconnected_cb": self.reconnected_cb,
# "error_cb": self.error_cb,
# "closed_cb": self.closed_cb,
"io_loop": self.loop,
}
self.loop.create_task(self.connect())
self.loop.run_forever()
def on_start(self):
"""
start the service
"""
LOGGER.debug("natsd.Service.on_start")
self.service = threading.Thread(target=self.run_event_loop, name=self.serviceQ + " service thread")
self.service.start()
while not self.is_started:
time.sleep(0.01)
def on_stop(self):
"""
stop the service
"""
LOGGER.debug("natsd.Service.on_stop")
self.is_started = False
try:
next(self.nc.unsubscribe(self.serviceQS))
except StopIteration as e:
pass
try:
next(self.nc.close())
except StopIteration as e:
pass
try:
for task in asyncio.Task.all_tasks(self.loop):
task.cancel()
self.loop.stop()
while self.loop.is_running():
time.sleep(1)
self.loop.close()
except Exception as e:
LOGGER.debug("natsd.Service.on_stop - Exception aio clean up : "
+ traceback.format_exc())
def on_failure(self, exception_type, exception_value, traceback_):
LOGGER.error("natsd.Requester.on_failure - " + exception_type.__str__() + "/" + exception_value.__str__())
LOGGER.error("natsd.Requester.on_failure - " + traceback_.format_exc())
self.is_started = False
try:
next(self.nc.unsubscribe(self.serviceQS))
except StopIteration as e:
pass
try:
next(self.nc.close())
except StopIteration as e:
pass
try:
for task in asyncio.Task.all_tasks(self.loop):
task.cancel()
self.loop.stop()
while self.loop.is_running():
time.sleep(1)
self.loop.close()
except Exception as e:
LOGGER.debug("natsd.Service.on_failure - Exception aio clean up : "
+ traceback.format_exc())
class Requester(pykka.ThreadingActor):
"""
NATS requester implementation
:param my_args: dict like {connection, request_q}
"""
def __init__(self, my_args=None, connection_args=None):
"""
NATS requester constructor
:param my_args: dict like {connection, request_q}
:param connection_args: dict like {user, password, host[, port, client_properties]}
:return: self
"""
LOGGER.debug("natsd.Requester.__init__")
if my_args is None:
raise exceptions.ArianeConfError("requestor arguments")
if 'request_q' not in my_args or my_args['request_q'] is None or not my_args['request_q']:
raise exceptions.ArianeConfError("request_q")
if 'fire_and_forget' not in my_args or my_args['fire_and_forget'] is None or not my_args['fire_and_forget']:
self.fire_and_forget = False
else:
self.fire_and_forget = True
if 'rpc_timeout' not in connection_args or connection_args['rpc_timeout'] is None or \
not connection_args['rpc_timeout']:
# default timeout = no timeout
self.rpc_timeout = 0
else:
self.rpc_timeout = connection_args['rpc_timeout']
if 'rpc_timeout_err_count_max' not in connection_args or connection_args['rpc_timeout_err_count_max'] is None \
or not connection_args['rpc_timeout_err_count_max']:
self.rpc_retry_timeout_err_count_max = 3
else:
self.rpc_retry_timeout_err_count_max = connection_args['rpc_timeout_err_count_max']
self.rpc_retry_timeout_err_count = 0
if 'rpc_retry' not in connection_args or connection_args['rpc_retry'] is None or \
not connection_args['rpc_retry']:
# default retry = no retry
self.rpc_retry = 0
else:
self.rpc_retry = connection_args['rpc_retry']
Driver.validate_driver_conf(connection_args)
super(Requester, self).__init__()
self.connection_args = copy.deepcopy(connection_args)
self.servers = [
"nats://" + connection_args['user'] + ":" + connection_args['password'] + "@" +
connection_args['host']+":"+str(connection_args['port'])
]
self.name = self.connection_args['client_properties']['ariane.app'] + "@" + socket.gethostname() + \
" - requestor on " + my_args['request_q']
self.loop = None
self.options = None
self.service = None
self.nc = Client()
self.requestQ = my_args['request_q']
self.responseQ = None
self.responseQS = None
self.response = None
self.split_responses = None
self.split_responses_mid = None
self.is_started = False
self.trace = False
self.max_payload = 0
if not self.fire_and_forget:
self.responseQ = new_inbox()
self.response = None
self.corr_id = None
def connect(self):
LOGGER.debug("natsd.Requester.connect")
try:
yield from self.nc.connect(**self.options)
if not self.fire_and_forget:
self.responseQS = yield from self.nc.subscribe(self.responseQ, cb=self.on_response)
self.max_payload = self.nc._max_payload
self.is_started = True
except ErrNoServers as e:
print(e)
return
def run_event_loop(self):
LOGGER.debug("natsd.Requester.run_event_loop")
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
self.options = {
"servers": self.servers,
"name": self.name,
# "disconnected_cb": self.disconnected_cb,
# "reconnected_cb": self.reconnected_cb,
# "error_cb": self.error_cb,
# "closed_cb": self.closed_cb,
"io_loop": self.loop,
}
self.loop.create_task(self.connect())
self.loop.run_forever()
def on_start(self):
"""
start requester
"""
LOGGER.debug("natsd.Requester.on_start")
self.service = threading.Thread(target=self.run_event_loop, name=self.requestQ + " requestor thread")
self.service.start()
while not self.is_started:
time.sleep(0.01)
def on_stop(self):
"""
stop requester
"""
LOGGER.debug("natsd.Requester.on_stop")
self.is_started = False
try:
LOGGER.debug("natsd.Requester.on_stop - unsubscribe from " + str(self.responseQS))
next(self.nc.unsubscribe(self.responseQS))
except StopIteration as e:
pass
try:
LOGGER.debug("natsd.Requester.on_stop - close nats connection")
next(self.nc.close())
except StopIteration as e:
pass
LOGGER.debug("natsd.Requester.on_stop - nc is closed: " + str(self.nc.is_closed))
try:
LOGGER.debug("natsd.Requester.on_stop - cancelling aio tasks loop")
loop_to_stop = self.loop
for task in asyncio.Task.all_tasks(loop_to_stop):
LOGGER.debug("natsd.Requester.on_stop - cancelling task " + str(task))
task.cancel()
LOGGER.debug("natsd.Requester.on_stop - stopping aio loop stop")
loop_to_stop.stop()
count = 0
while loop_to_stop.is_running():
count += 1
if count % 10 == 0:
LOGGER.debug("natsd.Requester.on_stop - waiting aio loop to be stopped (" +
str(asyncio.Task.all_tasks(loop_to_stop).__len__()) + " tasks left; " +
"current task: " + str(asyncio.Task.current_task(loop_to_stop)) + ")")
for task in asyncio.Task.all_tasks(loop_to_stop):
LOGGER.debug("natsd.Requester.on_stop - cancelling task " + str(task))
task.cancel()
time.sleep(1)
if count == 120:
LOGGER.error("natsd.Requester.on_stop - unable to stop aio loop after 120 sec (" +
str(asyncio.Task.all_tasks(loop_to_stop).__len__()) + " tasks left; " +
"current task: " + str(asyncio.Task.current_task(loop_to_stop)) + ")")
break
if not loop_to_stop.is_running():
LOGGER.debug("natsd.Requester.on_stop - close aio loop")
loop_to_stop.close()
except Exception as e:
LOGGER.warn("natsd.Requester.on_stop - exception on aio clean : "
+ traceback.format_exc())
def _restart_on_error(self):
LOGGER.debug("natsd.Requester._restart_on_error - restart begin !")
stop_thread = threading.Thread(target=self.on_stop, name=self.requestQ + " restarter.stop on error thread")
stop_thread.start()
while not self.nc.is_closed:
LOGGER.debug("natsd.Requester._restart_on_error - waiting nc to be closed")
time.sleep(1)
self.on_start()
self.rpc_retry_timeout_err_count = 0
LOGGER.debug("natsd.Requester._restart_on_error - restart end !")
def _restart_after_max_timeout_err_count(self):
restarter = threading.Thread(target=self._restart_on_error, name=self.requestQ + " restarter on error thread")
restarter.start()
def on_failure(self, exception_type, exception_value, traceback_):
LOGGER.error("natsd.Requester.on_failure - " + exception_type.__str__() + "/" + exception_value.__str__())
LOGGER.error("natsd.Requester.on_failure - " + traceback_.format_exc())
self.is_started = False
try:
next(self.nc.unsubscribe(self.responseQS))
except StopIteration as e:
pass
try:
next(self.nc.close())
except StopIteration as e:
pass
try:
loop_to_stop = self.loop
for task in asyncio.Task.all_tasks(loop_to_stop):
task.cancel()
loop_to_stop.stop()
while loop_to_stop.is_running():
time.sleep(1)
loop_to_stop.close()
except Exception as e:
LOGGER.debug("natsd.Requester.on_failure - exception on aio clean : "
+ traceback.format_exc())
def on_response(self, msg):
"""
setup response if correlation id is the good one
"""
LOGGER.debug("natsd.Requester.on_response: " + str(sys.getsizeof(msg)) + " bytes received")
working_response = json.loads(msg.data.decode())
working_properties = DriverTools.json2properties(working_response['properties'])
working_body = b''+bytes(working_response['body'], 'utf8') if 'body' in working_response else None
if DriverTools.MSG_CORRELATION_ID in working_properties:
if self.corr_id == working_properties[DriverTools.MSG_CORRELATION_ID]:
if DriverTools.MSG_SPLIT_COUNT in working_properties and \
int(working_properties[DriverTools.MSG_SPLIT_COUNT]) > 1:
working_body_decoded = base64.b64decode(working_body) if working_body is not None else None
if self.split_responses is None:
self.split_responses = []
self.split_responses_mid = working_properties[DriverTools.MSG_SPLIT_MID]
if working_properties[DriverTools.MSG_SPLIT_MID] == self.split_responses_mid:
response = {
'properties': working_properties,
'body': working_body_decoded
}
self.split_responses.insert(int(working_properties[DriverTools.MSG_SPLIT_OID]), response)
if self.split_responses.__len__() == int(working_properties[DriverTools.MSG_SPLIT_COUNT]):
properties = {}
body = b''
for num in range(0, self.split_responses.__len__()):
properties.update(self.split_responses[num]['properties'])
body += self.split_responses[num]['body']
self.response = {
'properties': properties,
'body': body
}
self.split_responses = None
self.split_responses_mid = None
else:
LOGGER.warn("natsd.Requester.on_response - discarded response : (" +
str(working_properties[DriverTools.MSG_CORRELATION_ID]) + "," +
str(working_properties[DriverTools.MSG_SPLIT_MID]) + ")")
LOGGER.debug("natsd.Requester.on_response - discarded response : " + str({
'properties': working_properties,
'body': working_body_decoded
}))
else:
working_body_decoded = base64.b64decode(working_body) if working_body is not None else \
bytes(json.dumps({}), 'utf8')
self.response = {
'properties': working_properties,
'body': working_body_decoded
}
else:
working_body_decoded = base64.b64decode(working_body) if working_body is not None else None
LOGGER.warn("natsd.Requester.on_response - discarded response : " +
str(working_properties[DriverTools.MSG_CORRELATION_ID]))
LOGGER.debug("natsd.Requester.on_response - discarded response : " + str({
'properties': working_properties,
'body': working_body_decoded
}))
else:
working_body_decoded = base64.b64decode(working_body) if working_body is not None else None
LOGGER.warn("natsd.Requester.on_response - discarded response (no correlation ID)")
LOGGER.debug("natsd.Requester.on_response - discarded response : " + str({
'properties': working_properties,
'body': working_body_decoded
}))
def _split_msg(self, split_mid, properties, body):
messages = []
in_progress_messages = []
msg_counter = 0
in_progress_properties_field = copy.deepcopy(properties)
if DriverTools.MSG_MESSAGE_ID in in_progress_properties_field:
in_progress_properties_field.pop(DriverTools.MSG_MESSAGE_ID)
if DriverTools.MSG_CORRELATION_ID in in_progress_properties_field:
in_progress_properties_field.pop(DriverTools.MSG_CORRELATION_ID)
if DriverTools.MSG_TRACE in in_progress_properties_field:
in_progress_properties_field.pop(DriverTools.MSG_TRACE)
if DriverTools.MSG_REPLY_TO in in_progress_properties_field:
in_progress_properties_field.pop(DriverTools.MSG_REPLY_TO)
wip_body = body
wip_body_len = sys.getsizeof(wip_body)
consumed_body_offset = 0
while (wip_body_len - consumed_body_offset) > 0 or in_progress_properties_field.__len__() > 0:
# consume properties first :
splitted_msg_size = 0
splitted_properties = {}
if DriverTools.MSG_MESSAGE_ID in properties:
splitted_properties[DriverTools.MSG_MESSAGE_ID] = properties[DriverTools.MSG_MESSAGE_ID]
if DriverTools.MSG_CORRELATION_ID in properties:
splitted_properties[DriverTools.MSG_CORRELATION_ID] = properties[DriverTools.MSG_CORRELATION_ID]
if DriverTools.MSG_TRACE in properties:
splitted_properties[DriverTools.MSG_TRACE] = properties[DriverTools.MSG_TRACE]
if DriverTools.MSG_REPLY_TO in properties:
splitted_properties[DriverTools.MSG_REPLY_TO] = properties[DriverTools.MSG_REPLY_TO]
splitted_properties[DriverTools.MSG_SPLIT_MID] = split_mid
splitted_properties[DriverTools.MSG_SPLIT_COUNT] = sys.maxsize
splitted_properties[DriverTools.MSG_SPLIT_OID] = msg_counter
splitted_typed_properties = None
for key, value in properties.items():
if key in in_progress_properties_field.keys():
splitted_properties[key] = value
tmp_splitted_typed_properties = []
for skey, svalue in splitted_properties.items():
tmp_splitted_typed_properties.append(DriverTools.property_params(skey, svalue))
msg_data = json.dumps({
'properties': tmp_splitted_typed_properties
})
msgb = b''+bytes(msg_data, 'utf8')
tmp_splitted_msg_size = sys.getsizeof(msgb)
if tmp_splitted_msg_size < self.max_payload:
splitted_typed_properties = tmp_splitted_typed_properties
in_progress_properties_field.pop(key)
else:
splitted_properties.pop(key)
msg_data = json.dumps({
'properties': splitted_typed_properties
})
msgb = b''+bytes(msg_data, 'utf8')
splitted_msg_size = sys.getsizeof(msgb)
# then body
splitted_body = None
if wip_body_len > 0:
chunk_size = self.max_payload - splitted_msg_size
if chunk_size > (wip_body_len - consumed_body_offset):
chunk_size = wip_body_len - consumed_body_offset
splitted_body = wip_body[consumed_body_offset:consumed_body_offset+chunk_size]
msg_data = json.dumps({
'properties': splitted_typed_properties,
'body': base64.b64encode(b''+bytes(splitted_body, 'utf8')).decode("utf-8")
})
msgb = b''+bytes(msg_data, 'utf8')
tmp_splitted_msg_size = sys.getsizeof(msgb)
while tmp_splitted_msg_size > self.max_payload:
chunk_size -= (tmp_splitted_msg_size - self.max_payload + 1)
splitted_body = wip_body[consumed_body_offset:consumed_body_offset+chunk_size]
msg_data = json.dumps({
'properties': splitted_typed_properties,
'body': base64.b64encode(b''+bytes(splitted_body, 'utf8')).decode("utf-8")
})
msgb = b''+bytes(msg_data, 'utf8')
tmp_splitted_msg_size = sys.getsizeof(msgb)
consumed_body_offset += chunk_size
# add splitted message into in_progress_messages
if splitted_body is not None:
in_progress_messages.append({
'properties': splitted_properties,
'body': base64.b64encode(b''+bytes(splitted_body, 'utf8')).decode("utf-8")
})
else:
in_progress_messages.append({
'properties': splitted_properties,
'body': ''
})
msg_counter += 1
for message in in_progress_messages:
message['properties'][DriverTools.MSG_SPLIT_COUNT] = msg_counter
typed_properties = []
for skey, svalue in message['properties'].items():
typed_properties.append(DriverTools.property_params(skey, svalue))
if 'body' in message:
msg_data = json.dumps({
'properties': typed_properties,
'body': message['body']
})
else:
msg_data = json.dumps({
'properties': typed_properties,
'body': ''
})
msgb = b''+bytes(msg_data, 'utf8')
messages.append(msgb)
return messages
def _init_split_msg_group(self, split_mid, msg_split_dest):
args = {'properties': {DriverTools.OPERATION_FDN: DriverTools.OP_MSG_SPLIT_FEED_INIT,
DriverTools.PARAM_MSG_SPLIT_MID: split_mid,
DriverTools.PARAM_MSG_SPLIT_FEED_DEST: msg_split_dest}}
fire_and_forget_changed = False
if self.fire_and_forget:
fire_and_forget_changed = True
self.fire_and_forget = False
previous_corr_id = self.corr_id
self.call(my_args=args)
self.response = None
self.corr_id = previous_corr_id
if fire_and_forget_changed:
self.fire_and_forget = True
def _end_split_msg_group(self, split_mid):
args = {'properties': {DriverTools.OPERATION_FDN: DriverTools.OP_MSG_SPLIT_FEED_END,
DriverTools.PARAM_MSG_SPLIT_MID: split_mid}}
fire_and_forget_changed = False
if self.fire_and_forget:
fire_and_forget_changed = True
self.fire_and_forget = False
previous_corr_id = self.corr_id
self.call(my_args=args)
self.response = None
self.corr_id = previous_corr_id
if fire_and_forget_changed:
self.fire_and_forget = True
def call(self, my_args=None):
"""
setup the request and call the remote service. Wait the answer (blocking call)
:param my_args: dict like {properties, body}
:return response
"""
if not self.is_started:
raise ArianeError('natsd.Requester.call',
'Requester not started !')
LOGGER.debug("natsd.Requester.call")
if my_args is None:
raise exceptions.ArianeConfError("requestor call arguments")
if 'properties' not in my_args or my_args['properties'] is None:
raise exceptions.ArianeConfError('requestor call properties')
if 'body' not in my_args or my_args['body'] is None:
my_args['body'] = ''
self.response = None
if not self.fire_and_forget:
if DriverTools.MSG_CORRELATION_ID not in my_args['properties']:
self.corr_id = str(uuid.uuid4())
properties = my_args['properties']
properties[DriverTools.MSG_CORRELATION_ID] = self.corr_id
else:
properties = my_args['properties']
self.corr_id = properties[DriverTools.MSG_CORRELATION_ID]
else:
properties = my_args['properties']
if 'sessionID' in properties and properties['sessionID'] is not None and properties['sessionID']:
request_q = str(properties['sessionID']) + '-' + self.requestQ
else:
request_q = self.requestQ
if self.trace:
properties[DriverTools.MSG_TRACE] = True
typed_properties = []
for key, value in properties.items():
typed_properties.append(DriverTools.property_params(key, value))
body = my_args['body']
if body:
body = base64.b64encode(b''+bytes(body, 'utf8')).decode("utf-8")
msg_data = json.dumps({
'properties': typed_properties,
'body': body
})
msgb = b''+bytes(msg_data, 'utf8')
split_mid = None
messages = []
if sys.getsizeof(msgb) > self.max_payload:
split_mid = str(uuid.uuid4())
messages = self._split_msg(split_mid, properties, my_args['body'])
else:
messages.append(msgb)
if not self.fire_and_forget:
if split_mid is not None and ('sessionID' not in properties or properties['sessionID'] is None or
not properties['sessionID']):
request_q += "_" + split_mid
self._init_split_msg_group(split_mid, request_q)
for msgb in messages:
try:
LOGGER.debug("natsd.Requester.call - publish splitted request " + str(typed_properties) +
" (size: " + str(sys.getsizeof(msgb)) + " bytes) on " + request_q)
next(self.nc.publish_request(request_q, self.responseQ, msgb))
except StopIteration as e:
pass
LOGGER.debug("natsd.Requester.call - waiting answer from " + self.responseQ)
else:
try:
LOGGER.debug("natsd.Requester.call - publish request " + str(typed_properties) + " on " + request_q)
next(self.nc.publish(request_q, b''+bytes(msg_data, 'utf8')))
except StopIteration as e:
pass
try:
next(self.nc.flush(1))
except StopIteration as e:
pass
start_time = timeit.default_timer()
if not self.fire_and_forget:
# Wait rpc_timeout sec before raising error
if self.rpc_timeout > 0:
exit_count = self.rpc_timeout * 100
else:
exit_count = 1
while self.response is None and exit_count > 0:
time.sleep(0.01)
if self.rpc_timeout > 0:
exit_count -= 1
if self.response is None:
if self.rpc_retry > 0:
if 'retry_count' not in my_args:
my_args['retry_count'] = 1
LOGGER.debug("natsd.Requester.call - Retry (" + str(my_args['retry_count']) + ")")
return self.call(my_args)
elif 'retry_count' in my_args and (self.rpc_retry - my_args['retry_count']) > 0:
LOGGER.warn("natsd.Requester.call - No response returned from request on " + request_q +
" queue after " + str(self.rpc_timeout) + '*' +
str(self.rpc_retry) + " sec ...")
self.trace = True
my_args['retry_count'] += 1
LOGGER.warn("natsd.Requester.call - Retry (" + str(my_args['retry_count']) + ")")
return self.call(my_args)
else:
self.rpc_retry_timeout_err_count += 1
if self.rpc_retry_timeout_err_count >= self.rpc_retry_timeout_err_count_max:
self._restart_after_max_timeout_err_count()
raise ArianeMessagingTimeoutError('natsd.Requester.call',
'Request timeout (' + str(self.rpc_timeout) + '*' +
str(self.rpc_retry) + ' sec) occured')
else:
self.rpc_retry_timeout_err_count += 1
if self.rpc_retry_timeout_err_count >= self.rpc_retry_timeout_err_count_max:
self._restart_after_max_timeout_err_count()
raise ArianeMessagingTimeoutError('natsd.Requester.call',
'Request timeout (' + str(self.rpc_timeout) + '*' +
str(self.rpc_retry) + ' sec) occured')
rpc_time = timeit.default_timer()-start_time
LOGGER.debug('natsd.Requester.call - RPC time : ' + str(rpc_time))
if self.rpc_timeout > 0 and rpc_time > self.rpc_timeout*3/5:
LOGGER.debug('natsd.Requester.call - slow RPC time (' + str(rpc_time) + ') on request ' +
str(typed_properties))
self.trace = False
self.rpc_retry_timeout_err_count = 0
rc_ = int(self.response['properties']['RC'])
if rc_ != 0:
try:
content = json.loads(self.response['body'].decode("UTF-8"))
except ValueError:
content = self.response['body'].decode("UTF-8")
dr = DriverResponse(
rc=rc_,
error_message=self.response['properties']['SERVER_ERROR_MESSAGE']
if 'SERVER_ERROR_MESSAGE' in self.response['properties'] else '',
response_content=content
)
else:
try:
if DriverTools.MSG_PROPERTIES in self.response['properties']:
props = json.loads(self.response['properties'][DriverTools.MSG_PROPERTIES])
else:
props = None
except ValueError:
if DriverTools.MSG_PROPERTIES in self.response['properties']:
props = self.response['props'][DriverTools.MSG_PROPERTIES]
else:
props = None
try:
content = json.loads(self.response['body'].decode("UTF-8"))
except ValueError:
content = self.response['body'].decode("UTF-8")
dr = DriverResponse(
rc=rc_,
response_properties=props,
response_content=content
)
if split_mid is not None and ('sessionID' not in properties or properties['sessionID'] is None or
not properties['sessionID']):
self._end_split_msg_group(split_mid)
request_q = request_q.split("_" + split_mid)[0]
return dr
class NATSHelper(object):
def __init__(self,
nats_broker_url=None,
nats_src_topic=None,
nats_dst_topic=None):
self.subscribe_id = None
self.connected = False
self.nats_client = NATS()
self._get_config_from_env_var_('nats_broker_url', 'NATS_ENDPOINT',
nats_broker_url, 'NATS broker')
self._get_config_from_env_var_('nats_src_topic', 'NATS_SRC_TOPIC',
nats_src_topic, 'NATS source topic')
self._get_config_from_env_var_('nats_dst_topic', 'NATS_DST_TOPIC',
nats_dst_topic,
'NATS destination topic')
logger.info("broker: {b}, src topic: {s}, dst_topic: {d}".format(
b=self.nats_broker_url,
s=self.nats_src_topic,
d=self.nats_dst_topic))
def __del__(self):
self.close()
def _get_config_from_env_var_(self, attr, var, default, what):
if default is None:
value = os.environ.get(var)
else:
value = default
if not value: # also check empty strings
raise Exception(
'{what} not provided in environment var {var}'.format(
what=what, var=var))
setattr(self, attr, value)
@staticmethod
def payload_from_message(msg):
"""
Convert input payload into an image
"""
_msg = xi_iot_pb2.DataStreamMessage()
_msg.ParseFromString(msg.data)
return _msg.payload
@staticmethod
def message_from_payload(payload):
"""
Convert image into output payload
"""
msg = xi_iot_pb2.DataStreamMessage()
msg.payload = payload
return msg.SerializeToString()
async def publish(self, payload):
try:
payload = self.message_from_payload(payload)
# RFC: We could leverage `reply` topic as the destination topic which would not require NATS_DST_TOPIC to be provided
# await nc.publish(reply, data)
logger.info("Sending message to topic '{}'".format(
self.nats_dst_topic))
await self.nats_client.publish(self.nats_dst_topic, payload)
except Exception as e:
# Catch an display errors which are otherwise not shown
logger.error("{}".format(e))
raise
async def connect(self, loop, message_handler_cb):
# Define a helper function to unbox the message and catch errors for display
async def receive_cb(msg):
try:
logger.info("Received a message!")
payload = self.payload_from_message(msg)
await message_handler_cb(payload)
except Exception as e:
# Catch an display errors which are otherwise not shown
logger.error("{}".format(e))
raise
try:
# This will return immediately if the server is not listening on the given URL
await self.nats_client.connect(self.nats_broker_url, loop=loop)
self.connected = True
logger.info(
"Connected to broker, subscribing to topic '{}'".format(
self.nats_src_topic))
self.subscribe_id = await self.nats_client.subscribe(
self.nats_src_topic, cb=receive_cb)
except Exception as e:
# Catch an display errors which are otherwise not shown
logger.error("{}".format(e))
raise
def close(self):
# Remove interest in subscription.
loop = asyncio.get_event_loop()
if self.subscribe_id is not None:
loop.run_until_complete(
self.nats_client.unsubscribe(self.subscribe_id))
self.subscribe_id = None
# Terminate connection to NATS.
if self.nats_client is not None and self.connected:
self.nats_client.drain()
loop.run_until_complete(self.nats_client.close())
self.nats_client = None
self.connected = False
