def run(loop):
nc = NATS()
options = {
"servers": [
"nats://tilient.org:44222",
"nats://wiffel.space:44222",
],
"io_loop": loop,
}
yield from nc.connect(**options)
yield from nc.publish("count", b"33")
yield from nc.flush(0.500)
yield from nc.publish("cmd", b"quit")
yield from nc.flush(0.500)
yield from nc.close()
def test_pending_data_size_flush_on_close(self):
nc = NATS()
disconnected_count = 0
reconnected_count = 0
closed_count = 0
err_count = 0
@asyncio.coroutine
def disconnected_cb():
nonlocal disconnected_count
disconnected_count += 1
@asyncio.coroutine
def reconnected_cb():
nonlocal reconnected_count
reconnected_count += 1
@asyncio.coroutine
def closed_cb():
nonlocal closed_count
closed_count += 1
options = {
'dont_randomize': True,
'io_loop': self.loop,
'disconnected_cb': disconnected_cb,
'closed_cb': closed_cb,
'reconnected_cb': reconnected_cb,
'reconnect_time_wait': 0.01
}
yield from nc.connect(**options)
total_received = 0
future = asyncio.Future(loop=self.loop)
@asyncio.coroutine
def receiver_cb(msg):
nonlocal total_received
total_received += 1
if total_received == 200:
future.set_result(True)
# Extra connection which should be receiving all the messages
nc2 = NATS()
yield from nc2.connect(**options)
yield from nc2.subscribe("example.*", cb=receiver_cb)
yield from nc2.flush()
for i in range(0, 200):
yield from nc.publish("example.{}".format(i), b'A' * 20)
# All pending messages should have been emitted to the server
# by the first connection at this point.
yield from nc.close()
# Wait for the server to flush all the messages back to the receiving client
yield from asyncio.wait_for(future, 1, loop=self.loop)
yield from nc2.close()
self.assertEqual(total_received, 200)
def go(loop):
nc = NATS()
try:
yield from nc.connect(io_loop=loop)
except:
pass
@asyncio.coroutine
def message_handler(msg):
print("[Received on '{}']: {}".format(msg.subject, msg.data.decode()))
try:
# Interested in receiving 2 messages from the 'discover' subject.
sid = yield from nc.subscribe("discover", "", message_handler)
yield from nc.auto_unsubscribe(sid, 2)
yield from nc.publish("discover", b'hello')
yield from nc.publish("discover", b'world')
# Following 2 messages won't be received.
yield from nc.publish("discover", b'again')
yield from nc.publish("discover", b'!!!!!')
except ErrConnectionClosed:
print("Connection closed prematurely")
@asyncio.coroutine
def request_handler(msg):
print("[Request on '{} {}']: {}".format(msg.subject, msg.reply, msg.data.decode()))
yield from nc.publish(msg.reply, b'OK')
if nc.is_connected:
# Subscription using a 'workers' queue so that only a single subscriber
# gets a request at a time.
yield from nc.subscribe("help", "workers", cb=request_handler)
try:
# Make a request expecting a single response within 500 ms,
# otherwise raising a timeout error.
msg = yield from nc.timed_request("help", b'help please', 0.500)
print("[Response]: {}".format(msg.data))
# Make a roundtrip to the server to ensure messages
# that sent messages have been processed already.
yield from nc.flush(0.500)
except ErrTimeout:
print("[Error] Timeout!")
# Wait a bit for message to be dispatched...
yield from asyncio.sleep(1, loop=loop)
# Detach from the server.
yield from nc.close()
if nc.last_error is not None:
print("Last Error: {}".format(nc.last_error))
if nc.is_closed:
print("Disconnected.")
def test_pending_data_size_flush_on_close(self):
nc = NATS()
disconnected_count = 0
reconnected_count = 0
closed_count = 0
err_count = 0
@asyncio.coroutine
def disconnected_cb():
nonlocal disconnected_count
disconnected_count += 1
@asyncio.coroutine
def reconnected_cb():
nonlocal reconnected_count
reconnected_count += 1
@asyncio.coroutine
def closed_cb():
nonlocal closed_count
closed_count += 1
options = {
'dont_randomize': True,
'io_loop': self.loop,
'disconnected_cb': disconnected_cb,
'closed_cb': closed_cb,
'reconnected_cb': reconnected_cb,
'reconnect_time_wait': 0.01
}
yield from nc.connect(**options)
total_received = 0
future = asyncio.Future(loop=self.loop)
@asyncio.coroutine
def receiver_cb(msg):
nonlocal total_received
total_received += 1
if total_received == 200:
future.set_result(True)
# Extra connection which should be receiving all the messages
nc2 = NATS()
yield from nc2.connect(**options)
yield from nc2.subscribe("example.*", cb=receiver_cb)
yield from nc2.flush()
for i in range(0, 200):
yield from nc.publish("example.{}".format(i), b'A' * 20)
# All pending messages should have been emitted to the server
# by the first connection at this point.
yield from nc.close()
# Wait for the server to flush all the messages back to the receiving client
yield from asyncio.wait_for(future, 1, loop=self.loop)
yield from nc2.close()
self.assertEqual(total_received, 200)
def main(loop):
parser = argparse.ArgumentParser()
parser.add_argument('-n', '--count', default=DEFAULT_NUM_MSGS, type=int)
parser.add_argument('-s', '--size', default=DEFAULT_MSG_SIZE, type=int)
parser.add_argument('-S', '--subject', default='test')
parser.add_argument('-b', '--batch', default=DEFAULT_BATCH_SIZE, type=int)
parser.add_argument('--servers', default=[], action='append')
args = parser.parse_args()
data = []
for i in range(0, args.size):
s = "%01x" % randint(0, 15)
data.append(s.encode())
payload = b''.join(data)
servers = args.servers
if len(args.servers) < 1:
servers = ["nats://127.0.0.1:4222"]
opts = { "servers": servers, "io_loop": loop }
# Make sure we're connected to a server first..
nc = NATS()
try:
yield from nc.connect(**opts)
except Exception as e:
sys.stderr.write("ERROR: {0}".format(e))
show_usage_and_die()
# Start the benchmark
start = time.time()
to_send = args.count
print("Sending {0} messages of size {1} bytes on [{2}]".format(
args.count, args.size, args.subject))
while to_send > 0:
for i in range(0, args.batch):
to_send -= 1
yield from nc.publish(args.subject, payload)
if (to_send % HASH_MODULO) == 0:
sys.stdout.write("#")
sys.stdout.flush()
if to_send == 0:
break
# Minimal pause in between batches sent to server
yield from asyncio.sleep(0.00001, loop=loop)
# Additional roundtrip with server to try to ensure everything has been sent already.
try:
yield from nc.flush(DEFAULT_FLUSH_TIMEOUT)
except ErrTimeout:
print("Server flush timeout after {0}".format(DEFAULT_FLUSH_TIMEOUT))
elapsed = time.time() - start
mbytes = "%.1f" % (((args.size * args.count)/elapsed) / (1024*1024))
print("\nTest completed : {0} msgs/sec ({1}) MB/sec".format(
args.count/elapsed,
mbytes))
yield from nc.close()
def main(loop):
parser = argparse.ArgumentParser()
parser.add_argument('-n', '--count', default=DEFAULT_NUM_MSGS, type=int)
parser.add_argument('-s', '--size', default=DEFAULT_MSG_SIZE, type=int)
parser.add_argument('-S', '--subject', default='test')
parser.add_argument('-b', '--batch', default=DEFAULT_BATCH_SIZE, type=int)
parser.add_argument('--servers', default=[], action='append')
args = parser.parse_args()
data = []
for i in range(0, args.size):
s = "%01x" % randint(0, 15)
data.append(s.encode())
payload = b''.join(data)
servers = args.servers
if len(args.servers) < 1:
servers = ["nats://127.0.0.1:4222"]
opts = {"servers": servers, "io_loop": loop}
# Make sure we're connected to a server first..
nc = NATS()
try:
yield from nc.connect(**opts)
except Exception as e:
sys.stderr.write("ERROR: {0}".format(e))
show_usage_and_die()
# Start the benchmark
start = time.time()
to_send = args.count
print("Sending {0} messages of size {1} bytes on [{2}]".format(
args.count, args.size, args.subject))
while to_send > 0:
for i in range(0, args.batch):
to_send -= 1
yield from nc.publish(args.subject, payload)
if (to_send % HASH_MODULO) == 0:
sys.stdout.write("#")
sys.stdout.flush()
if to_send == 0:
break
# Minimal pause in between batches sent to server
yield from asyncio.sleep(0.00001, loop=loop)
# Additional roundtrip with server to try to ensure everything has been sent already.
try:
yield from nc.flush(DEFAULT_FLUSH_TIMEOUT)
except ErrTimeout:
print("Server flush timeout after {0}".format(DEFAULT_FLUSH_TIMEOUT))
elapsed = time.time() - start
mbytes = "%.1f" % (((args.size * args.count) / elapsed) / (1024 * 1024))
print("\nTest completed : {0} msgs/sec ({1}) MB/sec".format(
args.count / elapsed, mbytes))
yield from nc.close()
def test_flush(self):
nc = NATS()
yield from nc.connect(io_loop=self.loop)
for i in range(0, 10):
yield from nc.publish("flush.%d" % i, b'AA')
yield from nc.flush()
self.assertEqual(10, nc.stats['out_msgs'])
self.assertEqual(20, nc.stats['out_bytes'])
yield from nc.close()
def test_flush(self):
nc = NATS()
yield from nc.connect(io_loop=self.loop)
for i in range(0, 10):
yield from nc.publish("flush.%d" % i, b'AA')
yield from nc.flush()
self.assertEqual(10, nc.stats['out_msgs'])
self.assertEqual(20, nc.stats['out_bytes'])
yield from nc.close()
def run(loop):
nc = NATS()
yield from nc.connect(servers=["nats://127.0.0.1:4222"], io_loop=loop)
yield from nc.publish("foo.thing", b'Hello!')
message = 'Current date: at {now}'.format(now=datetime.now().isoformat())
yield from nc.publish("foo.thing", message.encode())
yield from nc.flush()
yield from nc.close()
def test_close(self):
nc = NATS()
disconnected_count = 0
reconnected_count = 0
closed_count = 0
err_count = 0
@asyncio.coroutine
def disconnected_cb():
nonlocal disconnected_count
disconnected_count += 1
@asyncio.coroutine
def reconnected_cb():
nonlocal reconnected_count
reconnected_count += 1
@asyncio.coroutine
def closed_cb():
nonlocal closed_count
closed_count += 1
@asyncio.coroutine
def err_cb():
nonlocal err_count
err_count += 1
options = {
'io_loop': self.loop,
'disconnected_cb': disconnected_cb,
'closed_cb': closed_cb,
'reconnected_cb': reconnected_cb,
'error_cb': err_cb,
}
yield from nc.connect(**options)
yield from nc.close()
with self.assertRaises(ErrConnectionClosed):
yield from nc.publish("foo", b'A')
with self.assertRaises(ErrConnectionClosed):
yield from nc.subscribe("bar", "workers")
with self.assertRaises(ErrConnectionClosed):
yield from nc.publish_request("bar", "inbox", b'B')
with self.assertRaises(ErrConnectionClosed):
yield from nc.flush()
self.assertEqual(1, closed_count)
self.assertEqual(1, disconnected_count)
self.assertEqual(0, reconnected_count)
self.assertEqual(0, err_count)
def test_close(self):
nc = NATS()
disconnected_count = 0
reconnected_count = 0
closed_count = 0
err_count = 0
@asyncio.coroutine
def disconnected_cb():
nonlocal disconnected_count
disconnected_count += 1
@asyncio.coroutine
def reconnected_cb():
nonlocal reconnected_count
reconnected_count += 1
@asyncio.coroutine
def closed_cb():
nonlocal closed_count
closed_count += 1
@asyncio.coroutine
def err_cb(e):
nonlocal err_count
err_count += 1
options = {
'io_loop': self.loop,
'disconnected_cb': disconnected_cb,
'closed_cb': closed_cb,
'reconnected_cb': reconnected_cb,
'error_cb': err_cb,
}
yield from nc.connect(**options)
yield from nc.close()
with self.assertRaises(ErrConnectionClosed):
yield from nc.publish("foo", b'A')
with self.assertRaises(ErrConnectionClosed):
yield from nc.subscribe("bar", "workers")
with self.assertRaises(ErrConnectionClosed):
yield from nc.publish_request("bar", "inbox", b'B')
with self.assertRaises(ErrConnectionClosed):
yield from nc.flush()
self.assertEqual(1, closed_count)
self.assertEqual(1, disconnected_count)
self.assertEqual(0, reconnected_count)
self.assertEqual(0, err_count)
def run(loop):
nc = NATS()
nats_connection_string = "nats://*****:*****@bike.dowhile.se:4222"
position_bus = {}
position_bus["lat"] = 57.709548
position_bus["lon"] = 11.941056
position_bus["speed"] = 12.5
position_bus["course"] = 45
position_bus["time"] = datetime.datetime.now(
datetime.timezone.utc).isoformat()
position_bus["vehicle_type"] = "bus"
position_bike = {}
position_bike["lat"] = 57.709627
position_bike["lon"] = 11.942357
position_bike["speed"] = 14.5
position_bike["course"] = 330
position_bike["time"] = datetime.datetime.now(
datetime.timezone.utc).isoformat()
position_bike["vehicle_type"] = "bicycle"
try:
yield from nc.connect(io_loop=loop, servers=[nats_connection_string])
except ErrNoServers as e:
print(e)
return
@asyncio.coroutine
def message_sender(position, vehicle_id):
subject = "vehicle." + vehicle_id + ".position"
yield from nc.publish(subject, json.dumps(position).encode())
@asyncio.coroutine
def message_printer(msg):
subject = msg.subject
reply = msg.reply
data = msg.data.decode()
print("Received a message on '{subject} {reply}': {data}".format(
subject=subject, reply=reply, data=data))
sys.stdout.flush()
yield from nc.subscribe("*.>", cb=message_printer)
yield from message_sender(position_bus, "testBUS1")
yield from message_sender(position_bike, "testBIKE1")
try:
# Flush connection to server, returns when all messages have been processed.
# It raises a timeout if roundtrip takes longer than 1 second.
yield from nc.flush(1)
except ErrTimeout:
print("Flush timeout")
yield from asyncio.sleep(1, loop=loop)
yield from nc.close()
def main(loop):
parser = argparse.ArgumentParser()
parser.add_argument('-n', '--count', default=DEFAULT_NUM_MSGS, type=int)
parser.add_argument('-S', '--subject', default='test')
parser.add_argument('--servers', default=[], action='append')
args = parser.parse_args()
servers = args.servers
if len(args.servers) < 1:
servers = ["nats://127.0.0.1:4222"]
opts = { "servers": servers, "io_loop": loop, "allow_reconnect": False }
# Make sure we're connected to a server first...
nc = NATS()
try:
yield from nc.connect(**opts)
except Exception as e:
sys.stderr.write("ERROR: {0}".format(e))
show_usage_and_die()
received = 0
start = None
@asyncio.coroutine
def handler(msg):
nonlocal received
nonlocal start
received += 1
# Measure time from when we get the first message.
if received == 1:
start = time.monotonic()
if (received % HASH_MODULO) == 0:
sys.stdout.write("*")
sys.stdout.flush()
yield from nc.subscribe(args.subject, cb=handler)
# Additional roundtrip with server to ensure everything has been
# processed by the server already.
yield from nc.flush()
print("Waiting for {} messages on [{}]...".format(args.count, args.subject))
try:
while received < args.count:
yield from asyncio.sleep(0.1, loop=loop)
except ErrTimeout:
print("Server flush timeout after {0}".format(DEFAULT_FLUSH_TIMEOUT))
elapsed = time.monotonic() - start
print("\nTest completed : {0} msgs/sec sent".format(args.count/elapsed))
print("Received {0} messages ({1} msgs/sec)".format(received, received/elapsed))
yield from nc.close()
def publish_msg(self, subject, msg, loop):
client = NATS()
servers = [self.url]
options = {
"io_loop": loop,
"servers": servers,
}
try:
yield from client.connect(**options)
yield from client.publish(subject, msg)
yield from client.flush()
yield from client.close()
except Exception as e:
print(e)
pass
def run(loop):
parser = argparse.ArgumentParser()
# e.g. nats-pub hello -d "world" -s nats://127.0.0.1:4222 -s nats://127.0.0.1:4223
parser.add_argument('subject', default='hello', nargs='?')
parser.add_argument('-d', '--data', default="hello world")
parser.add_argument('-s', '--servers', default=[], action='append')
args = parser.parse_args()
nc = NATS()
@asyncio.coroutine
def closed_cb():
print("Connection to NATS is closed.")
loop.stop()
@asyncio.coroutine
def reconnected_cb():
print("Connected to NATS at {}...".format(nc.connected_url.netloc))
@asyncio.coroutine
def subscribe_handler(msg):
subject = msg.subject
reply = msg.reply
data = msg.data.decode()
print("Received a message on '{subject} {reply}': {data}".format(
subject=subject, reply=reply, data=data))
options = {
"io_loop": loop,
"closed_cb": closed_cb,
"reconnected_cb": reconnected_cb
}
try:
if len(args.servers) > 0:
options['servers'] = args.servers
yield from nc.connect(**options)
except Exception as e:
print(e)
show_usage_and_die()
print("Connected to NATS at {}...".format(nc.connected_url.netloc))
yield from nc.publish(args.subject, args.data.encode())
yield from nc.flush()
yield from nc.close()
def run(loop):
parser = argparse.ArgumentParser()
# e.g. nats-pub hello -d "world" -s nats://127.0.0.1:4222 -s nats://127.0.0.1:4223
parser.add_argument('subject', default='hello', nargs='?')
parser.add_argument('-d', '--data', default="hello world")
parser.add_argument('-s', '--servers', default=[], action='append')
parser.add_argument('--creds', default="")
args = parser.parse_args()
nc = NATS()
@asyncio.coroutine
def error_cb(e):
print("Error:", e)
@asyncio.coroutine
def closed_cb():
print("Connection to NATS is closed.")
@asyncio.coroutine
def reconnected_cb():
print("Connected to NATS at {}...".format(nc.connected_url.netloc))
options = {
"io_loop": loop,
"error_cb": error_cb,
"closed_cb": closed_cb,
"reconnected_cb": reconnected_cb
}
if len(args.creds) > 0:
options["user_credentials"] = args.creds
try:
if len(args.servers) > 0:
options['servers'] = args.servers
yield from nc.connect(**options)
except Exception as e:
print(e)
show_usage_and_die()
print("Connected to NATS at {}...".format(nc.connected_url.netloc))
yield from nc.publish(args.subject, args.data.encode())
yield from nc.flush()
yield from nc.close()
def run(loop):
nc = NATS()
try:
yield from nc.connect(servers=["nats://127.0.0.1:4222"], io_loop=loop)
except ErrNoServers as e:
print(e)
return
@asyncio.coroutine
def message_handler(msg):
subject = msg.subject
reply = msg.reply
data = msg.data.decode()
for i in range(0, 20):
yield from nc.publish(reply, "i={i}".format(i=i).encode())
yield from nc.subscribe("help.>", cb=message_handler)
@asyncio.coroutine
def request_handler(msg):
subject = msg.subject
reply = msg.reply
data = msg.data.decode()
print("Received a message on '{subject} {reply}': {data}".format(
subject=subject, reply=reply, data=data))
# Signal the server to stop sending messages after we got 10 already.
yield from nc.request("help.please",
b'help',
expected=10,
cb=request_handler)
try:
# Flush connection to server, returns when all messages have been processed.
# It raises a timeout if roundtrip takes longer than 1 second.
yield from nc.flush(1)
except ErrTimeout:
print("Flush timeout")
yield from asyncio.sleep(1, loop=loop)
yield from nc.close()
def test_publish(self):
nc = NATS()
yield from nc.connect(io_loop=self.loop)
for i in range(0, 100):
yield from nc.publish("hello.%d" % i, b'A')
with self.assertRaises(ErrBadSubject):
yield from nc.publish("", b'')
yield from nc.flush()
yield from nc.close()
yield from asyncio.sleep(1, loop=self.loop)
self.assertEqual(100, nc.stats['out_msgs'])
self.assertEqual(100, nc.stats['out_bytes'])
endpoint = '127.0.0.1:{port}'.format(port=self.server_pool[0].http_port)
httpclient = http.client.HTTPConnection(endpoint, timeout=5)
httpclient.request('GET', '/varz')
response = httpclient.getresponse()
varz = json.loads((response.read()).decode())
self.assertEqual(100, varz['in_msgs'])
self.assertEqual(100, varz['in_bytes'])
def run(loop):
nc = NATS()
try:
yield from nc.connect(io_loop=loop)
except ErrNoServers as e:
print(e)
return
@asyncio.coroutine
def message_handler(msg):
subject = msg.subject
reply = msg.reply
data = msg.data.decode()
for i in range(0, 20):
yield from nc.publish(reply, "i={i}".format(i=i).encode())
yield from nc.subscribe("help.>", cb=message_handler)
@asyncio.coroutine
def request_handler(msg):
subject = msg.subject
reply = msg.reply
data = msg.data.decode()
print("Received a message on '{subject} {reply}': {data}".format(
subject=subject, reply=reply, data=data))
# Signal the server to stop sending messages after we got 10 already.
yield from nc.request("help.please", b'help', expected=10, cb=request_handler)
try:
# Flush connection to server, returns when all messages have been processed.
# It raises a timeout if roundtrip takes longer than 1 second.
yield from nc.flush(1)
except ErrTimeout:
print("Flush timeout")
yield from asyncio.sleep(1, loop=loop)
yield from nc.close()
def test_publish(self):
nc = NATS()
yield from nc.connect(io_loop=self.loop)
for i in range(0, 100):
yield from nc.publish("hello.%d" % i, b'A')
with self.assertRaises(ErrBadSubject):
yield from nc.publish("", b'')
yield from nc.flush()
yield from nc.close()
yield from asyncio.sleep(1, loop=self.loop)
self.assertEqual(100, nc.stats['out_msgs'])
self.assertEqual(100, nc.stats['out_bytes'])
endpoint = '127.0.0.1:{port}'.format(
port=self.server_pool[0].http_port)
httpclient = http.client.HTTPConnection(endpoint, timeout=5)
httpclient.request('GET', '/varz')
response = httpclient.getresponse()
varz = json.loads((response.read()).decode())
self.assertEqual(100, varz['in_msgs'])
self.assertEqual(100, varz['in_bytes'])
def run(loop):
nc = NATS()
@asyncio.coroutine
def closed_cb():
logging.info("Connection to NATS is closed.")
yield from asyncio.sleep(0.1, loop=loop)
loop.stop()
options = {"servers": [natsURL], "io_loop": loop, "closed_cb": closed_cb}
yield from nc.connect(**options)
logging.info("Connected to NATS at {}...".format(nc.connected_url.netloc))
def signal_handler():
if nc.is_closed:
return
logging.info("Disconnecting...")
loop.create_task(nc.close())
for sig in ('SIGINT', 'SIGTERM'):
loop.add_signal_handler(getattr(signal, sig), signal_handler)
while (True):
dt = datetime.now(JST)
timestamp = dt.strftime("%Y-%m-%d %H:%M:%S:%f")
status = str(random.randint(0, 1))
msg = json.dumps({"timestamp": timestamp, "status": status})
logging.info("Publishing to NATS topic: {}".format(topic))
logging.info("Publishing msg: {}".format(msg.encode()))
yield from nc.publish(topic, msg.encode())
yield from nc.flush()
time.sleep(10)
yield from nc.close()
def test_custom_flush_queue_reconnect(self):
nc = NATS()
disconnected_count = 0
reconnected_count = 0
closed_count = 0
err_count = 0
@asyncio.coroutine
def disconnected_cb():
nonlocal disconnected_count
disconnected_count += 1
@asyncio.coroutine
def reconnected_cb():
nonlocal reconnected_count
reconnected_count += 1
@asyncio.coroutine
def closed_cb():
nonlocal closed_count
closed_count += 1
options = {
'servers': [
"nats://*****:*****@127.0.0.1:4223",
"nats://*****:*****@127.0.0.1:4224"
],
'dont_randomize': True,
'io_loop': self.loop,
'disconnected_cb': disconnected_cb,
'closed_cb': closed_cb,
'reconnected_cb': reconnected_cb,
'flusher_queue_size': 100,
'reconnect_time_wait': 0.01
}
yield from nc.connect(**options)
largest_pending_data_size = 0
post_flush_pending_data = None
done_once = False
@asyncio.coroutine
def cb(msg):
pass
yield from nc.subscribe("example.*", cb=cb)
for i in range(0, 500):
yield from nc.publish("example.{}".format(i), b'A' * 20)
if nc.pending_data_size > 0:
largest_pending_data_size = nc.pending_data_size
if nc.pending_data_size > 100:
# Stop the first server and connect to another one asap.
if not done_once:
yield from nc.flush(2)
post_flush_pending_data = nc.pending_data_size
yield from self.loop.run_in_executor(None, self.server_pool[0].stop)
done_once = True
self.assertTrue(largest_pending_data_size > 0)
self.assertTrue(post_flush_pending_data == 0)
# Confirm we have reconnected eventually
for i in range(0, 10):
yield from asyncio.sleep(0, loop=self.loop)
yield from asyncio.sleep(0.2, loop=self.loop)
yield from asyncio.sleep(0, loop=self.loop)
self.assertEqual(1, nc.stats['reconnects'])
try:
yield from nc.flush(2)
except ErrTimeout:
# If disconnect occurs during this flush, then we will have a timeout here
pass
finally:
yield from nc.close()
self.assertTrue(disconnected_count >= 1)
self.assertTrue(closed_count >= 1)
def go(loop):
nc = NATS()
try:
yield from nc.connect(io_loop=loop)
except:
pass
@asyncio.coroutine
def message_handler(msg):
print("[Received on '{}']: {}".format(msg.subject, msg.data.decode()))
try:
# Interested in receiving 2 messages from the 'discover' subject.
sid = yield from nc.subscribe("discover", "", message_handler)
yield from nc.auto_unsubscribe(sid, 2)
yield from nc.publish("discover", b'hello')
yield from nc.publish("discover", b'world')
# Following 2 messages won't be received.
yield from nc.publish("discover", b'again')
yield from nc.publish("discover", b'!!!!!')
except ErrConnectionClosed:
print("Connection closed prematurely")
@asyncio.coroutine
def request_handler(msg):
print("[Request on '{} {}']: {}".format(msg.subject, msg.reply,
msg.data.decode()))
yield from nc.publish(msg.reply, b'OK')
if nc.is_connected:
# Subscription using a 'workers' queue so that only a single subscriber
# gets a request at a time.
yield from nc.subscribe("help", "workers", cb=request_handler)
try:
# Make a request expecting a single response within 500 ms,
# otherwise raising a timeout error.
msg = yield from nc.timed_request("help", b'help please', 0.500)
print("[Response]: {}".format(msg.data))
# Make a roundtrip to the server to ensure messages
# that sent messages have been processed already.
yield from nc.flush(0.500)
except ErrTimeout:
print("[Error] Timeout!")
# Wait a bit for message to be dispatched...
yield from asyncio.sleep(1, loop=loop)
# Detach from the server.
yield from nc.close()
if nc.last_error is not None:
print("Last Error: {}".format(nc.last_error))
if nc.is_closed:
print("Disconnected.")
class NatsIo(object):
"""
asyncio NATS client
"""
def __init__(self,
host='127.0.0.1',
port='4222',
streaming_id=None,
cluster='svom-cluster'):
self.nc = NATS()
self.server = host
self.is_connected = False
self.streaming_id = streaming_id
self.cluster = cluster
self.sc = None
self.loop = None
try:
self.loop = asyncio.get_event_loop()
except RuntimeError:
LOG.info(
'Failed to retrieve asyncio event loop. Creating a new one...')
self.loop = asyncio.new_event_loop()
self.loop.run_until_complete(self._connect(servers=[self.server]))
if streaming_id is not None:
if has_streaming is True:
self.sc = STAN()
self.loop.run_until_complete(self._stan_connect())
else:
LOG.error("Can't run client without asyncio-nats-streaming")
sys.exit(1)
self.nats_server = NATSClient(self.nc, self.loop)
self.nats_server.start()
def is_streaming(self):
return True if self.streaming_id is not None else False
def _connect(self, servers):
"""connect"""
while not self.is_connected:
try:
LOG.info('Attempting connection to nats server ' + servers[0])
yield from self.nc.connect(servers=servers, io_loop=self.loop)
yield from self.nc.flush()
except Exception:
LOG.exception('Connection failed.')
LOG.info('Retrying connection in 5s.')
yield from asyncio.sleep(5)
else:
self.is_connected = True
LOG.info('Connected.')
if self.streaming_id is not None:
self.is_connected = False
@asyncio.coroutine
def _stan_connect(self):
"""connect"""
while not self.is_connected:
try:
LOG.info('Attempting connection to server cluster \'' +
self.cluster + '\' with clientID \'' +
self.streaming_id + '\'')
yield from self.sc.connect(self.cluster,
self.streaming_id,
nats=self.nc)
except Exception:
LOG.exception('Connection failed.')
LOG.info('Retrying streaming server connection in 5s.')
yield from asyncio.sleep(5)
else:
self.is_connected = True
LOG.info('Connected to streaming server.')
def subscribe(self, subject, handler=None):
"""launch subscription as async task"""
if not self.is_connected:
LOG.info("Awaiting to be connected to subscribe")
self.loop.call_later(5, self.subscribe, subject, handler)
if handler is None:
handler = self._default_handler
if self.streaming_id is None:
asyncio.run_coroutine_threadsafe(self._subscribe(subject, handler),
loop=self.loop)
else:
asyncio.run_coroutine_threadsafe(self._stan_subscribe(
subject, handler),
loop=self.loop)
@asyncio.coroutine
def _subscribe(self, subject, handler):
"""new subscription or handler change"""
LOG.info('Subscribing to subject \'' + subject + '\'')
try:
yield from self.nc.subscribe(subject, cb=handler)
except Exception:
LOG.exception('Subscription failed.')
else:
LOG.info('Subscribed to subject \'' + subject + '\'')
@asyncio.coroutine
def _stan_subscribe(self, subject, handler):
"""new subscription or handler change"""
LOG.info('Subscribing to subject \'' + subject + '\'')
try:
# No need to custom durable name since 'clientID+durable name' is checked
yield from self.sc.subscribe(subject,
durable_name=handler.__name__,
cb=handler)
except Exception as e:
LOG.exception('Subscription to \'%s\' failed' % subject)
else:
LOG.info('Handler \'%s\' subscribed to subject \'%s\'' %
(handler.__name__, subject))
def publish(self, subject, data, with_reply=False):
"""launch publish as async task"""
if isinstance(data, str):
data = data.encode()
if with_reply is False:
if self.streaming_id is None:
asyncio.run_coroutine_threadsafe(self._publish(subject, data),
loop=self.loop)
else:
asyncio.run_coroutine_threadsafe(self._stan_publish(subject,\
data), loop=self.loop)
else:
if self.streaming_id is None:
future = asyncio.run_coroutine_threadsafe(\
self._publish(subject,\
data, with_reply=True), loop=self.loop)
try:
result = future.result(1)
if result is None:
LOG.error('Message received no response')
return None
except Exception:
LOG.exception('Timed request publishing exception')
else:
return result
else:
LOG.error('NATS Streaming publish cannot handle replies')
@asyncio.coroutine
def _publish(self, subject, data, with_reply=False):
"""Nats transport data : bytes or str"""
try:
if with_reply is False:
yield from self.nc.publish(subject, data)
else:
yield from self.nc.timed_request(subject, data)
except Exception as e:
LOG.exception('Publication failed.')
@asyncio.coroutine
def _stan_publish(self, subject, data):
"""Nats transport data : bytes or str"""
try:
yield from self.sc.publish(subject,\
data, ack_handler=self._ack_handler)
except Exception as e:
LOG.exception('Publication failed, is \'' + \
subject + '\' a valid channel?')
@asyncio.coroutine
def _default_handler(self, msg):
"""Application handler data : str"""
LOG.debug("--- Received: " + msg.subject + msg.data.decode())
if self.streaming_id is not None:
yield from self.sc.ack(msg)
@asyncio.coroutine
def ack(self, msg):
yield from self.sc.ack(msg)
@asyncio.coroutine
def _ack_handler(self, ack):
LOG.debug('Delivery ack received: {}'.format(ack.guid))
def stop(self):
LOG.debug('Trying to stop properly NatsIo: ')
asyncio.run_coroutine_threadsafe(self.nc.flush(), loop=self.loop)
asyncio.run_coroutine_threadsafe(self.nc.close(), loop=self.loop)
try:
time.sleep(1)
self.loop.call_soon_threadsafe(self.loop.stop)
except Exception as e:
LOG.debug(e)
try:
time.sleep(1)
if self.loop.is_running():
LOG.debug('loop.is_running: ' + str(self.loop.is_running()))
self.loop.call_soon_threadsafe(self.loop.close())
except Exception as e:
LOG.debug(e)
self.nats_server.join()
def main(loop):
parser = argparse.ArgumentParser()
parser.add_argument('-n', '--count', default=DEFAULT_NUM_MSGS, type=int)
parser.add_argument('-S', '--subject', default='test')
parser.add_argument('-t', '--subtype', default='sync')
parser.add_argument('--servers', default=[], action='append')
args = parser.parse_args()
servers = args.servers
if len(args.servers) < 1:
servers = ["nats://127.0.0.1:4222"]
opts = { "servers": servers, "io_loop": loop, "allow_reconnect": False }
# Make sure we're connected to a server first...
nc = NATS()
try:
yield from nc.connect(**opts)
except Exception as e:
sys.stderr.write("ERROR: {0}".format(e))
show_usage_and_die()
received = 0
start = None
@asyncio.coroutine
def handler(msg):
nonlocal received
nonlocal start
received += 1
# Measure time from when we get the first message.
if received == 1:
start = time.monotonic()
if (received % HASH_MODULO) == 0:
sys.stdout.write("*")
sys.stdout.flush()
if args.subtype == 'sync':
yield from nc.subscribe(args.subject, cb=handler)
elif args.subtype == 'async':
yield from nc.subscribe_async(args.subject, cb=handler)
else:
sys.stderr.write("ERROR: Unsupported type of subscription {0}".format(e))
show_usage_and_die()
print("Waiting for {} messages on [{}]...".format(args.count, args.subject))
try:
# Additional roundtrip with server to ensure everything has been
# processed by the server already.
yield from nc.flush()
except ErrTimeout:
print("Server flush timeout after {0}".format(DEFAULT_FLUSH_TIMEOUT))
while received < args.count:
yield from asyncio.sleep(0.1, loop=loop)
elapsed = time.monotonic() - start
print("\nTest completed : {0} msgs/sec sent".format(args.count/elapsed))
print("Received {0} messages ({1} msgs/sec)".format(received, received/elapsed))
yield from nc.close()
class Requester(pykka.ThreadingActor):
"""
NATS requester implementation
:param my_args: dict like {connection, request_q}
"""
def __init__(self, my_args=None, connection_args=None):
"""
NATS requester constructor
:param my_args: dict like {connection, request_q}
:param connection_args: dict like {user, password, host[, port, client_properties]}
:return: self
"""
LOGGER.debug("natsd.Requester.__init__")
if my_args is None:
raise exceptions.ArianeConfError("requestor arguments")
if 'request_q' not in my_args or my_args['request_q'] is None or not my_args['request_q']:
raise exceptions.ArianeConfError("request_q")
if 'fire_and_forget' not in my_args or my_args['fire_and_forget'] is None or not my_args['fire_and_forget']:
self.fire_and_forget = False
else:
self.fire_and_forget = True
if 'rpc_timeout' not in connection_args or connection_args['rpc_timeout'] is None or \
not connection_args['rpc_timeout']:
# default timeout = no timeout
self.rpc_timeout = 0
else:
self.rpc_timeout = connection_args['rpc_timeout']
if 'rpc_timeout_err_count_max' not in connection_args or connection_args['rpc_timeout_err_count_max'] is None \
or not connection_args['rpc_timeout_err_count_max']:
self.rpc_retry_timeout_err_count_max = 3
else:
self.rpc_retry_timeout_err_count_max = connection_args['rpc_timeout_err_count_max']
self.rpc_retry_timeout_err_count = 0
if 'rpc_retry' not in connection_args or connection_args['rpc_retry'] is None or \
not connection_args['rpc_retry']:
# default retry = no retry
self.rpc_retry = 0
else:
self.rpc_retry = connection_args['rpc_retry']
Driver.validate_driver_conf(connection_args)
super(Requester, self).__init__()
self.connection_args = copy.deepcopy(connection_args)
self.servers = [
"nats://" + connection_args['user'] + ":" + connection_args['password'] + "@" +
connection_args['host']+":"+str(connection_args['port'])
]
self.name = self.connection_args['client_properties']['ariane.app'] + "@" + socket.gethostname() + \
" - requestor on " + my_args['request_q']
self.loop = None
self.options = None
self.service = None
self.nc = Client()
self.requestQ = my_args['request_q']
self.responseQ = None
self.responseQS = None
self.response = None
self.split_responses = None
self.split_responses_mid = None
self.is_started = False
self.trace = False
self.max_payload = 0
if not self.fire_and_forget:
self.responseQ = new_inbox()
self.response = None
self.corr_id = None
def connect(self):
LOGGER.debug("natsd.Requester.connect")
try:
yield from self.nc.connect(**self.options)
if not self.fire_and_forget:
self.responseQS = yield from self.nc.subscribe(self.responseQ, cb=self.on_response)
self.max_payload = self.nc._max_payload
self.is_started = True
except ErrNoServers as e:
print(e)
return
def run_event_loop(self):
LOGGER.debug("natsd.Requester.run_event_loop")
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
self.options = {
"servers": self.servers,
"name": self.name,
# "disconnected_cb": self.disconnected_cb,
# "reconnected_cb": self.reconnected_cb,
# "error_cb": self.error_cb,
# "closed_cb": self.closed_cb,
"io_loop": self.loop,
}
self.loop.create_task(self.connect())
self.loop.run_forever()
def on_start(self):
"""
start requester
"""
LOGGER.debug("natsd.Requester.on_start")
self.service = threading.Thread(target=self.run_event_loop, name=self.requestQ + " requestor thread")
self.service.start()
while not self.is_started:
time.sleep(0.01)
def on_stop(self):
"""
stop requester
"""
LOGGER.debug("natsd.Requester.on_stop")
self.is_started = False
try:
LOGGER.debug("natsd.Requester.on_stop - unsubscribe from " + str(self.responseQS))
next(self.nc.unsubscribe(self.responseQS))
except StopIteration as e:
pass
try:
LOGGER.debug("natsd.Requester.on_stop - close nats connection")
next(self.nc.close())
except StopIteration as e:
pass
LOGGER.debug("natsd.Requester.on_stop - nc is closed: " + str(self.nc.is_closed))
try:
LOGGER.debug("natsd.Requester.on_stop - cancelling aio tasks loop")
loop_to_stop = self.loop
for task in asyncio.Task.all_tasks(loop_to_stop):
LOGGER.debug("natsd.Requester.on_stop - cancelling task " + str(task))
task.cancel()
LOGGER.debug("natsd.Requester.on_stop - stopping aio loop stop")
loop_to_stop.stop()
count = 0
while loop_to_stop.is_running():
count += 1
if count % 10 == 0:
LOGGER.debug("natsd.Requester.on_stop - waiting aio loop to be stopped (" +
str(asyncio.Task.all_tasks(loop_to_stop).__len__()) + " tasks left; " +
"current task: " + str(asyncio.Task.current_task(loop_to_stop)) + ")")
for task in asyncio.Task.all_tasks(loop_to_stop):
LOGGER.debug("natsd.Requester.on_stop - cancelling task " + str(task))
task.cancel()
time.sleep(1)
if count == 120:
LOGGER.error("natsd.Requester.on_stop - unable to stop aio loop after 120 sec (" +
str(asyncio.Task.all_tasks(loop_to_stop).__len__()) + " tasks left; " +
"current task: " + str(asyncio.Task.current_task(loop_to_stop)) + ")")
break
if not loop_to_stop.is_running():
LOGGER.debug("natsd.Requester.on_stop - close aio loop")
loop_to_stop.close()
except Exception as e:
LOGGER.warn("natsd.Requester.on_stop - exception on aio clean : "
+ traceback.format_exc())
def _restart_on_error(self):
LOGGER.debug("natsd.Requester._restart_on_error - restart begin !")
stop_thread = threading.Thread(target=self.on_stop, name=self.requestQ + " restarter.stop on error thread")
stop_thread.start()
while not self.nc.is_closed:
LOGGER.debug("natsd.Requester._restart_on_error - waiting nc to be closed")
time.sleep(1)
self.on_start()
self.rpc_retry_timeout_err_count = 0
LOGGER.debug("natsd.Requester._restart_on_error - restart end !")
def _restart_after_max_timeout_err_count(self):
restarter = threading.Thread(target=self._restart_on_error, name=self.requestQ + " restarter on error thread")
restarter.start()
def on_failure(self, exception_type, exception_value, traceback_):
LOGGER.error("natsd.Requester.on_failure - " + exception_type.__str__() + "/" + exception_value.__str__())
LOGGER.error("natsd.Requester.on_failure - " + traceback_.format_exc())
self.is_started = False
try:
next(self.nc.unsubscribe(self.responseQS))
except StopIteration as e:
pass
try:
next(self.nc.close())
except StopIteration as e:
pass
try:
loop_to_stop = self.loop
for task in asyncio.Task.all_tasks(loop_to_stop):
task.cancel()
loop_to_stop.stop()
while loop_to_stop.is_running():
time.sleep(1)
loop_to_stop.close()
except Exception as e:
LOGGER.debug("natsd.Requester.on_failure - exception on aio clean : "
+ traceback.format_exc())
def on_response(self, msg):
"""
setup response if correlation id is the good one
"""
LOGGER.debug("natsd.Requester.on_response: " + str(sys.getsizeof(msg)) + " bytes received")
working_response = json.loads(msg.data.decode())
working_properties = DriverTools.json2properties(working_response['properties'])
working_body = b''+bytes(working_response['body'], 'utf8') if 'body' in working_response else None
if DriverTools.MSG_CORRELATION_ID in working_properties:
if self.corr_id == working_properties[DriverTools.MSG_CORRELATION_ID]:
if DriverTools.MSG_SPLIT_COUNT in working_properties and \
int(working_properties[DriverTools.MSG_SPLIT_COUNT]) > 1:
working_body_decoded = base64.b64decode(working_body) if working_body is not None else None
if self.split_responses is None:
self.split_responses = []
self.split_responses_mid = working_properties[DriverTools.MSG_SPLIT_MID]
if working_properties[DriverTools.MSG_SPLIT_MID] == self.split_responses_mid:
response = {
'properties': working_properties,
'body': working_body_decoded
}
self.split_responses.insert(int(working_properties[DriverTools.MSG_SPLIT_OID]), response)
if self.split_responses.__len__() == int(working_properties[DriverTools.MSG_SPLIT_COUNT]):
properties = {}
body = b''
for num in range(0, self.split_responses.__len__()):
properties.update(self.split_responses[num]['properties'])
body += self.split_responses[num]['body']
self.response = {
'properties': properties,
'body': body
}
self.split_responses = None
self.split_responses_mid = None
else:
LOGGER.warn("natsd.Requester.on_response - discarded response : (" +
str(working_properties[DriverTools.MSG_CORRELATION_ID]) + "," +
str(working_properties[DriverTools.MSG_SPLIT_MID]) + ")")
LOGGER.debug("natsd.Requester.on_response - discarded response : " + str({
'properties': working_properties,
'body': working_body_decoded
}))
else:
working_body_decoded = base64.b64decode(working_body) if working_body is not None else \
bytes(json.dumps({}), 'utf8')
self.response = {
'properties': working_properties,
'body': working_body_decoded
}
else:
working_body_decoded = base64.b64decode(working_body) if working_body is not None else None
LOGGER.warn("natsd.Requester.on_response - discarded response : " +
str(working_properties[DriverTools.MSG_CORRELATION_ID]))
LOGGER.debug("natsd.Requester.on_response - discarded response : " + str({
'properties': working_properties,
'body': working_body_decoded
}))
else:
working_body_decoded = base64.b64decode(working_body) if working_body is not None else None
LOGGER.warn("natsd.Requester.on_response - discarded response (no correlation ID)")
LOGGER.debug("natsd.Requester.on_response - discarded response : " + str({
'properties': working_properties,
'body': working_body_decoded
}))
def _split_msg(self, split_mid, properties, body):
messages = []
in_progress_messages = []
msg_counter = 0
in_progress_properties_field = copy.deepcopy(properties)
if DriverTools.MSG_MESSAGE_ID in in_progress_properties_field:
in_progress_properties_field.pop(DriverTools.MSG_MESSAGE_ID)
if DriverTools.MSG_CORRELATION_ID in in_progress_properties_field:
in_progress_properties_field.pop(DriverTools.MSG_CORRELATION_ID)
if DriverTools.MSG_TRACE in in_progress_properties_field:
in_progress_properties_field.pop(DriverTools.MSG_TRACE)
if DriverTools.MSG_REPLY_TO in in_progress_properties_field:
in_progress_properties_field.pop(DriverTools.MSG_REPLY_TO)
wip_body = body
wip_body_len = sys.getsizeof(wip_body)
consumed_body_offset = 0
while (wip_body_len - consumed_body_offset) > 0 or in_progress_properties_field.__len__() > 0:
# consume properties first :
splitted_msg_size = 0
splitted_properties = {}
if DriverTools.MSG_MESSAGE_ID in properties:
splitted_properties[DriverTools.MSG_MESSAGE_ID] = properties[DriverTools.MSG_MESSAGE_ID]
if DriverTools.MSG_CORRELATION_ID in properties:
splitted_properties[DriverTools.MSG_CORRELATION_ID] = properties[DriverTools.MSG_CORRELATION_ID]
if DriverTools.MSG_TRACE in properties:
splitted_properties[DriverTools.MSG_TRACE] = properties[DriverTools.MSG_TRACE]
if DriverTools.MSG_REPLY_TO in properties:
splitted_properties[DriverTools.MSG_REPLY_TO] = properties[DriverTools.MSG_REPLY_TO]
splitted_properties[DriverTools.MSG_SPLIT_MID] = split_mid
splitted_properties[DriverTools.MSG_SPLIT_COUNT] = sys.maxsize
splitted_properties[DriverTools.MSG_SPLIT_OID] = msg_counter
splitted_typed_properties = None
for key, value in properties.items():
if key in in_progress_properties_field.keys():
splitted_properties[key] = value
tmp_splitted_typed_properties = []
for skey, svalue in splitted_properties.items():
tmp_splitted_typed_properties.append(DriverTools.property_params(skey, svalue))
msg_data = json.dumps({
'properties': tmp_splitted_typed_properties
})
msgb = b''+bytes(msg_data, 'utf8')
tmp_splitted_msg_size = sys.getsizeof(msgb)
if tmp_splitted_msg_size < self.max_payload:
splitted_typed_properties = tmp_splitted_typed_properties
in_progress_properties_field.pop(key)
else:
splitted_properties.pop(key)
msg_data = json.dumps({
'properties': splitted_typed_properties
})
msgb = b''+bytes(msg_data, 'utf8')
splitted_msg_size = sys.getsizeof(msgb)
# then body
splitted_body = None
if wip_body_len > 0:
chunk_size = self.max_payload - splitted_msg_size
if chunk_size > (wip_body_len - consumed_body_offset):
chunk_size = wip_body_len - consumed_body_offset
splitted_body = wip_body[consumed_body_offset:consumed_body_offset+chunk_size]
msg_data = json.dumps({
'properties': splitted_typed_properties,
'body': base64.b64encode(b''+bytes(splitted_body, 'utf8')).decode("utf-8")
})
msgb = b''+bytes(msg_data, 'utf8')
tmp_splitted_msg_size = sys.getsizeof(msgb)
while tmp_splitted_msg_size > self.max_payload:
chunk_size -= (tmp_splitted_msg_size - self.max_payload + 1)
splitted_body = wip_body[consumed_body_offset:consumed_body_offset+chunk_size]
msg_data = json.dumps({
'properties': splitted_typed_properties,
'body': base64.b64encode(b''+bytes(splitted_body, 'utf8')).decode("utf-8")
})
msgb = b''+bytes(msg_data, 'utf8')
tmp_splitted_msg_size = sys.getsizeof(msgb)
consumed_body_offset += chunk_size
# add splitted message into in_progress_messages
if splitted_body is not None:
in_progress_messages.append({
'properties': splitted_properties,
'body': base64.b64encode(b''+bytes(splitted_body, 'utf8')).decode("utf-8")
})
else:
in_progress_messages.append({
'properties': splitted_properties,
'body': ''
})
msg_counter += 1
for message in in_progress_messages:
message['properties'][DriverTools.MSG_SPLIT_COUNT] = msg_counter
typed_properties = []
for skey, svalue in message['properties'].items():
typed_properties.append(DriverTools.property_params(skey, svalue))
if 'body' in message:
msg_data = json.dumps({
'properties': typed_properties,
'body': message['body']
})
else:
msg_data = json.dumps({
'properties': typed_properties,
'body': ''
})
msgb = b''+bytes(msg_data, 'utf8')
messages.append(msgb)
return messages
def _init_split_msg_group(self, split_mid, msg_split_dest):
args = {'properties': {DriverTools.OPERATION_FDN: DriverTools.OP_MSG_SPLIT_FEED_INIT,
DriverTools.PARAM_MSG_SPLIT_MID: split_mid,
DriverTools.PARAM_MSG_SPLIT_FEED_DEST: msg_split_dest}}
fire_and_forget_changed = False
if self.fire_and_forget:
fire_and_forget_changed = True
self.fire_and_forget = False
previous_corr_id = self.corr_id
self.call(my_args=args)
self.response = None
self.corr_id = previous_corr_id
if fire_and_forget_changed:
self.fire_and_forget = True
def _end_split_msg_group(self, split_mid):
args = {'properties': {DriverTools.OPERATION_FDN: DriverTools.OP_MSG_SPLIT_FEED_END,
DriverTools.PARAM_MSG_SPLIT_MID: split_mid}}
fire_and_forget_changed = False
if self.fire_and_forget:
fire_and_forget_changed = True
self.fire_and_forget = False
previous_corr_id = self.corr_id
self.call(my_args=args)
self.response = None
self.corr_id = previous_corr_id
if fire_and_forget_changed:
self.fire_and_forget = True
def call(self, my_args=None):
"""
setup the request and call the remote service. Wait the answer (blocking call)
:param my_args: dict like {properties, body}
:return response
"""
if not self.is_started:
raise ArianeError('natsd.Requester.call',
'Requester not started !')
LOGGER.debug("natsd.Requester.call")
if my_args is None:
raise exceptions.ArianeConfError("requestor call arguments")
if 'properties' not in my_args or my_args['properties'] is None:
raise exceptions.ArianeConfError('requestor call properties')
if 'body' not in my_args or my_args['body'] is None:
my_args['body'] = ''
self.response = None
if not self.fire_and_forget:
if DriverTools.MSG_CORRELATION_ID not in my_args['properties']:
self.corr_id = str(uuid.uuid4())
properties = my_args['properties']
properties[DriverTools.MSG_CORRELATION_ID] = self.corr_id
else:
properties = my_args['properties']
self.corr_id = properties[DriverTools.MSG_CORRELATION_ID]
else:
properties = my_args['properties']
if 'sessionID' in properties and properties['sessionID'] is not None and properties['sessionID']:
request_q = str(properties['sessionID']) + '-' + self.requestQ
else:
request_q = self.requestQ
if self.trace:
properties[DriverTools.MSG_TRACE] = True
typed_properties = []
for key, value in properties.items():
typed_properties.append(DriverTools.property_params(key, value))
body = my_args['body']
if body:
body = base64.b64encode(b''+bytes(body, 'utf8')).decode("utf-8")
msg_data = json.dumps({
'properties': typed_properties,
'body': body
})
msgb = b''+bytes(msg_data, 'utf8')
split_mid = None
messages = []
if sys.getsizeof(msgb) > self.max_payload:
split_mid = str(uuid.uuid4())
messages = self._split_msg(split_mid, properties, my_args['body'])
else:
messages.append(msgb)
if not self.fire_and_forget:
if split_mid is not None and ('sessionID' not in properties or properties['sessionID'] is None or
not properties['sessionID']):
request_q += "_" + split_mid
self._init_split_msg_group(split_mid, request_q)
for msgb in messages:
try:
LOGGER.debug("natsd.Requester.call - publish splitted request " + str(typed_properties) +
" (size: " + str(sys.getsizeof(msgb)) + " bytes) on " + request_q)
next(self.nc.publish_request(request_q, self.responseQ, msgb))
except StopIteration as e:
pass
LOGGER.debug("natsd.Requester.call - waiting answer from " + self.responseQ)
else:
try:
LOGGER.debug("natsd.Requester.call - publish request " + str(typed_properties) + " on " + request_q)
next(self.nc.publish(request_q, b''+bytes(msg_data, 'utf8')))
except StopIteration as e:
pass
try:
next(self.nc.flush(1))
except StopIteration as e:
pass
start_time = timeit.default_timer()
if not self.fire_and_forget:
# Wait rpc_timeout sec before raising error
if self.rpc_timeout > 0:
exit_count = self.rpc_timeout * 100
else:
exit_count = 1
while self.response is None and exit_count > 0:
time.sleep(0.01)
if self.rpc_timeout > 0:
exit_count -= 1
if self.response is None:
if self.rpc_retry > 0:
if 'retry_count' not in my_args:
my_args['retry_count'] = 1
LOGGER.debug("natsd.Requester.call - Retry (" + str(my_args['retry_count']) + ")")
return self.call(my_args)
elif 'retry_count' in my_args and (self.rpc_retry - my_args['retry_count']) > 0:
LOGGER.warn("natsd.Requester.call - No response returned from request on " + request_q +
" queue after " + str(self.rpc_timeout) + '*' +
str(self.rpc_retry) + " sec ...")
self.trace = True
my_args['retry_count'] += 1
LOGGER.warn("natsd.Requester.call - Retry (" + str(my_args['retry_count']) + ")")
return self.call(my_args)
else:
self.rpc_retry_timeout_err_count += 1
if self.rpc_retry_timeout_err_count >= self.rpc_retry_timeout_err_count_max:
self._restart_after_max_timeout_err_count()
raise ArianeMessagingTimeoutError('natsd.Requester.call',
'Request timeout (' + str(self.rpc_timeout) + '*' +
str(self.rpc_retry) + ' sec) occured')
else:
self.rpc_retry_timeout_err_count += 1
if self.rpc_retry_timeout_err_count >= self.rpc_retry_timeout_err_count_max:
self._restart_after_max_timeout_err_count()
raise ArianeMessagingTimeoutError('natsd.Requester.call',
'Request timeout (' + str(self.rpc_timeout) + '*' +
str(self.rpc_retry) + ' sec) occured')
rpc_time = timeit.default_timer()-start_time
LOGGER.debug('natsd.Requester.call - RPC time : ' + str(rpc_time))
if self.rpc_timeout > 0 and rpc_time > self.rpc_timeout*3/5:
LOGGER.debug('natsd.Requester.call - slow RPC time (' + str(rpc_time) + ') on request ' +
str(typed_properties))
self.trace = False
self.rpc_retry_timeout_err_count = 0
rc_ = int(self.response['properties']['RC'])
if rc_ != 0:
try:
content = json.loads(self.response['body'].decode("UTF-8"))
except ValueError:
content = self.response['body'].decode("UTF-8")
dr = DriverResponse(
rc=rc_,
error_message=self.response['properties']['SERVER_ERROR_MESSAGE']
if 'SERVER_ERROR_MESSAGE' in self.response['properties'] else '',
response_content=content
)
else:
try:
if DriverTools.MSG_PROPERTIES in self.response['properties']:
props = json.loads(self.response['properties'][DriverTools.MSG_PROPERTIES])
else:
props = None
except ValueError:
if DriverTools.MSG_PROPERTIES in self.response['properties']:
props = self.response['props'][DriverTools.MSG_PROPERTIES]
else:
props = None
try:
content = json.loads(self.response['body'].decode("UTF-8"))
except ValueError:
content = self.response['body'].decode("UTF-8")
dr = DriverResponse(
rc=rc_,
response_properties=props,
response_content=content
)
if split_mid is not None and ('sessionID' not in properties or properties['sessionID'] is None or
not properties['sessionID']):
self._end_split_msg_group(split_mid)
request_q = request_q.split("_" + split_mid)[0]
return dr
def run(loop):
nc = NATS()
# Setup pool of servers from a NATS cluster.
options = {
"servers": [
"nats://*****:*****@127.0.0.1:4222",
"nats://*****:*****@127.0.0.1:4223",
"nats://*****:*****@127.0.0.1:4224",
],
"io_loop":
loop,
}
# Auth token can be used instead if setting
# as part of uri string of a server:
# "nats://[email protected]:4222"
# Will try to connect to servers in order of configuration,
# by defaults it connect to one in the pool randomly.
options["dont_randomize"] = True
# Optionally set reconnect wait and max reconnect attempts.
# This example means 10 seconds total per backend.
options["max_reconnect_attempts"] = 5
options["reconnect_time_wait"] = 2
@asyncio.coroutine
def disconnected_cb():
print("Got disconnected!")
@asyncio.coroutine
def reconnected_cb():
# See who we are connected to on reconnect.
print("Got reconnected to {url}".format(url=nc.connected_url.netloc))
# Setup callbacks to be notified on disconnects and reconnects
options["disconnected_cb"] = disconnected_cb
options["reconnected_cb"] = reconnected_cb
@asyncio.coroutine
def error_cb(e):
print("There was an error: {}".format(e))
@asyncio.coroutine
def closed_cb():
print("Connection is closed")
@asyncio.coroutine
def subscribe_handler(msg):
print("Got message: ", msg.subject, msg.reply, msg.data)
# Setup callbacks to be notified when there is an error
# or connection is closed.
options["error_cb"] = error_cb
options["closed_cb"] = closed_cb
try:
yield from nc.connect(**options)
except ErrNoServers as e:
# Could not connect to any server in the cluster.
print(e)
return
if nc.is_connected:
yield from nc.subscribe("help.*", cb=subscribe_handler)
#max_messages = 1000000
max_messages = 10
start_time = datetime.now()
print("Sending {} messages to NATS...".format(max_messages))
for i in range(0, max_messages):
try:
yield from nc.publish("help.{}".format(i), b'A')
yield from nc.flush(0.500)
except ErrConnectionClosed as e:
print("Connection closed prematurely.")
break
except ErrTimeout as e:
print("Timeout occured when publishing msg i={}: {}".format(
i, e))
end_time = datetime.now()
yield from nc.close()
duration = end_time - start_time
print("Duration: {}".format(duration))
try:
yield from nc.publish("help", b"hello world")
except ErrConnectionClosed:
print("Can't publish since no longer connected.")
err = nc.last_error
if err is not None:
print("Last Error: {}".format(err))
def test_custom_flush_queue_reconnect(self):
nc = NATS()
disconnected_count = 0
reconnected_count = 0
closed_count = 0
err_count = 0
@asyncio.coroutine
def disconnected_cb():
nonlocal disconnected_count
disconnected_count += 1
@asyncio.coroutine
def reconnected_cb():
nonlocal reconnected_count
reconnected_count += 1
@asyncio.coroutine
def closed_cb():
nonlocal closed_count
closed_count += 1
options = {
'servers': [
"nats://*****:*****@127.0.0.1:4223",
"nats://*****:*****@127.0.0.1:4224"
],
'dont_randomize':
True,
'io_loop':
self.loop,
'disconnected_cb':
disconnected_cb,
'closed_cb':
closed_cb,
'reconnected_cb':
reconnected_cb,
'flusher_queue_size':
100,
'reconnect_time_wait':
0.01
}
yield from nc.connect(**options)
largest_pending_data_size = 0
post_flush_pending_data = None
done_once = False
@asyncio.coroutine
def cb(msg):
pass
yield from nc.subscribe("example.*", cb=cb)
for i in range(0, 500):
yield from nc.publish("example.{}".format(i), b'A' * 20)
if nc.pending_data_size > 0:
largest_pending_data_size = nc.pending_data_size
if nc.pending_data_size > 100:
# Stop the first server and connect to another one asap.
if not done_once:
yield from nc.flush(2)
post_flush_pending_data = nc.pending_data_size
yield from self.loop.run_in_executor(
None, self.server_pool[0].stop)
done_once = True
self.assertTrue(largest_pending_data_size > 0)
self.assertTrue(post_flush_pending_data == 0)
# Confirm we have reconnected eventually
for i in range(0, 10):
yield from asyncio.sleep(0, loop=self.loop)
yield from asyncio.sleep(0.2, loop=self.loop)
yield from asyncio.sleep(0, loop=self.loop)
self.assertEqual(1, nc.stats['reconnects'])
try:
yield from nc.flush(2)
except ErrTimeout:
# If disconnect occurs during this flush, then we will have a timeout here
pass
finally:
yield from nc.close()
self.assertTrue(disconnected_count >= 1)
self.assertTrue(closed_count >= 1)
class _AsyncIONats(threading.Thread):
"""Tiny class to handle queuing requests through asyncio.
Essentially, a wrapper class around the Nats.IO asyncio implementation to provide us with
the functionality we're after. This makes for a much nicer interface to work with than
the incredibly annoying & ugly examples https://github.com/nats-io/asyncio-nats .. ewww.
"""
_MAX_RECONNECTS = 10
def __init__(self, url, tls):
threading.Thread.__init__(self)
self._conn = None
self._outgoing = queue.Queue() # outbound messages added in request()
self._running = False
self.opts = { # opts to pass to Nats.io client
"servers": [url],
"allow_reconnect": True,
"max_reconnect_attempts": self._MAX_RECONNECTS,
}
if tls:
self.opts["tls"] = tls
@asyncio.coroutine
def main(self, loop):
"""Connect to remote host(s)
Raises:
NoServersError
"""
# explicitly set the asyncio event loop so it can't get confused ..
asyncio.set_event_loop(loop)
self._conn = NatsClient()
try:
yield from self._conn.connect(io_loop=loop, **self.opts)
except nats_errors.ErrNoServers as e:
# Could not connect to any server in the cluster.
raise errors.NoServersError(e)
while self._running:
if self._outgoing.empty():
# No one wants to send a message
continue
if not self._conn.is_connected:
# give nats more time to (re)connect
continue
reply_queue, key, data = self._outgoing.get_nowait(
) # pull request from queue
if reply_queue is None:
# we're passed None only when we're supposed to exit. See stop()
break
try:
result = yield from self._conn.request(
key, bytes(data, encoding="utf8"))
reply_queue.put_nowait(result.data.decode())
except nats_errors.ErrConnectionClosed as e:
reply_queue.put_nowait(errors.ConnectionClosedError(e))
except (nats_errors.ErrTimeout, queue.Empty) as e:
reply_queue.put_nowait(errors.RequestTimeoutError(e))
except Exception as e: # pass all errors up to the caller
reply_queue.put_nowait(e)
yield from self._conn.close()
def request(self, data: dict, key: str, timeout: int = 5) -> dict:
"""Send a request to the server & await the reply.
Args:
data: data to send
key: the key (subject) to send the message to
timeout: some time in seconds to wait before calling it quits
Returns:
dict
Raises:
RequestTimeoutError
ConnectionClosedError
NoServersError
"""
q = queue.Queue(
maxsize=NATS_MSG_RETRIES) # create a queue to get a reply on
self._outgoing.put_nowait(
(q, key, data)) # add our message to the outbound queue
try:
result = q.get(timeout=max([_NATS_MIN_TIMEOUT,
timeout])) # block for a reply
except queue.Empty as e: # we waited, but nothing was returned to us :(
raise errors.RequestTimeoutError(
"Timeout waiting for server reply. Original %s" % e)
if isinstance(result, Exception):
raise result
return result
def stop(self):
"""Stop the service, killing open connection(s)
"""
# set to false to kill coroutine running in main()
self._running = False
# interpreted as a poison pill (causes main() loop to break)
self._outgoing.put((None, None, None))
if not self._conn:
return
try:
# flush & kill the actual connections
self._conn.flush()
self._conn.close()
except Exception:
pass
def run(self):
"""Start the service
"""
if self._running:
return
self._running = True
loop = asyncio.new_event_loop()
loop.run_until_complete(self.main(loop))
try:
loop.close()
except Exception:
pass
def run(loop):
nc = NATS()
# Setup pool of servers from a NATS cluster.
options = {
"servers": [
"nats://*****:*****@127.0.0.1:4222",
"nats://*****:*****@127.0.0.1:4223",
"nats://*****:*****@127.0.0.1:4224",
],
"io_loop": loop,
}
# Will try to connect to servers in order of configuration,
# by defaults it connect to one in the pool randomly.
options["dont_randomize"] = True
# Optionally set reconnect wait and max reconnect attempts.
# This example means 10 seconds total per backend.
options["max_reconnect_attempts"] = 5
options["reconnect_time_wait"] = 2
@asyncio.coroutine
def disconnected_cb():
print("Got disconnected!")
@asyncio.coroutine
def reconnected_cb():
# See who we are connected to on reconnect.
print("Got reconnected to {url}".format(url=nc.connected_url.netloc))
# Setup callbacks to be notified on disconnects and reconnects
options["disconnected_cb"] = disconnected_cb
options["reconnected_cb"] = reconnected_cb
@asyncio.coroutine
def error_cb(e):
print("There was an error: {}".format(e))
@asyncio.coroutine
def closed_cb():
print("Connection is closed")
# Setup callbacks to be notified when there is an error
# or connection is closed.
options["error_cb"] = error_cb
options["closed_cb"] = closed_cb
try:
yield from nc.connect(**options)
except ErrNoServers as e:
# Could not connect to any server in the cluster.
print(e)
return
if nc.is_connected:
yield from nc.subscribe("help.*")
max_messages = 1000000
start_time = datetime.now()
print("Sending {} messages to NATS...".format(max_messages))
for i in range(0, max_messages):
try:
yield from nc.publish("help.{}".format(i), b'A')
yield from nc.flush(0.500)
except ErrConnectionClosed as e:
print("Connection closed prematurely.")
break
except ErrTimeout as e:
print("Timeout occured when publishing msg i={}: {}".format(i, e))
end_time = datetime.now()
yield from nc.close()
duration = end_time - start_time
print("Duration: {}".format(duration))
try:
yield from nc.publish("help", b"hello world")
except ErrConnectionClosed:
print("Can't publish since no longer connected.")
err = nc.last_error
if err is not None:
print("Last Error: {}".format(err))
