async def test_connecting_with_dup_id(self):
nc = NATS()
await nc.connect(loop=self.loop)
sc = STAN()
client_id = generate_client_id()
await sc.connect("test-cluster", client_id, nats=nc)
sc_2 = STAN()
with self.assertRaises(StanError):
await sc_2.connect("test-cluster", client_id, nats=nc)
# Publish a some messages
msgs = []
future = asyncio.Future(loop=self.loop)
async def cb(msg):
nonlocal msgs
msgs.append(msg)
if len(msgs) == 10:
future.set_result(True)
# Start a subscription and wait to receive all the messages
# which have been sent so far.
sub = await sc.subscribe("hi", cb=cb)
for i in range(0, 10):
await sc.publish("hi", b'hello')
try:
await asyncio.wait_for(future, 2, loop=self.loop)
except:
pass
self.assertEqual(len(msgs), 10)
for i in range(0, 10):
m = msgs[i]
self.assertEqual(m.sequence, i + 1)
# Need to cleanup STAN session before wrapping up NATS conn.
await sc.close()
# Should have removed acks and HBs subscriptions.
self.assertEqual(len(nc._subs), 1)
scs = [sc, sc_2]
for s in scs:
self.assertEqual(s._hb_inbox, None)
self.assertEqual(s._hb_inbox_sid, None)
self.assertEqual(s._ack_subject, None)
self.assertEqual(s._ack_subject_sid, None)
await nc.close()
self.assertFalse(nc.is_connected)
async def run(loop):
#
nc = NATS()
sc = STAN()
options = {
"servers": ["nats://192.168.100.228:4222"],
"io_loop": loop,
"error_cb": error_cb
}
await nc.connect(**options)
# Start session with NATS Streaming cluster using
# the established NATS connection.
await sc.connect("vsblty-cluster", "client-1235487854", nats=nc)
async def cb(msg):
task1 = await loop.create_task(FrameProces(msg.seq, msg.data))
# Subscribe to get all messages from the beginning.
await sc.subscribe("VSBLTY-DATA-TEMP", start_at='first', cb=cb)
def run():
# Use borrowed connection for NATS then mount NATS Streaming
# client on top.
nc = NATS()
nc.connect()
# Start session with NATS Streaming cluster.
sc = STAN()
sc.connect("test-cluster", "client-123", nats=nc)
total_messages = 0
#future = asyncio.Future(loop=loop)
def queue_cb(msg):
#nonlocal future
nonlocal total_messages
print("Received a message (seq={}): {}".format(msg.seq, msg.data.decode()))
total_messages += 1
print(total_messages)
r = binary_to_dict(msg.data.decode())
print(r)
# if total_messages >= 2:
# future.set_result(None)
# Subscribe to get all messages since beginning.
sub = sc.subscribe("hi", queue="bar", cb=queue_cb)
async def connect(self):
"""Connect the service worker to th3e NATS/STAN Queue.
Also handles reconnecting when the network has dropped the connection.
Both the NATS and the STAN clients need to be reinstantiated to work correctly.
"""
if not self.config:
logger.error('missing configuration object.')
raise AttributeError('missing configuration object.')
logger.info('Connecting...')
if self.nc:
try:
logger.debug('close old NATS client')
await self.nc.close()
except asyncio.CancelledError as err:
logger.debug('closing stale connection err:%s', err)
finally:
self.nc = None
self.nc = NATS()
self.sc = STAN()
nats_connection_options = {
**self.config.NATS_CONNECTION_OPTIONS,
**{
'loop': self._loop,
'error_cb': error_cb
},
**self.nats_connection_options
}
stan_connection_options = {
**self.config.STAN_CONNECTION_OPTIONS,
**{
'nats': self.nc,
'conn_lost_cb': self._stan_conn_lost_cb,
'loop': self._loop
},
**self.stan_connection_options
}
subscription_options = {
**self.config.SUBSCRIPTION_OPTIONS,
**{
'cb': self.cb_handler
},
**self.subscription_options
}
await self.nc.connect(**nats_connection_options)
await self.sc.connect(**stan_connection_options)
await self.sc.subscribe(**subscription_options)
logger.info(
'Subscribe the callback: %s to the queue: %s.',
subscription_options.get('cb').__name__
if subscription_options.get('cb') else 'no_call_back',
subscription_options.get('queue'))
async def run(loop):
nc = NATS()
await nc.connect(servers=["nats://localhost:4222"], io_loop=loop)
sc = STAN()
await sc.connect("test-cluster", "dataeng-publisher", nats=nc)
# sebelumnya kita publish angka saja
# i = 0
# while True:
# print(f"publish {i}")
# await sc.publish("hitung", str(i).encode())
# i += 1
# time.sleep(1)
# lalu kita ingin publish message dalam bentuk protocol buffers
write_card_holder = WriteCardHolder()
while True:
card_holder_book = holder_pb2.CardHolder()
msg = write_card_holder.PromptForAddress(card_holder_book)
print("Publish", msg)
await sc.publish("hitung", str(msg).encode())
time.sleep(1)
def signal_handler():
if nc.is_closed:
return
print('Disconnecting...')
loop.create_task(sc.close())
loop.create_task(nc.close())
for sig in ('SIGINT', 'SIGTERM'):
loop.add_signal_handler(getattr(signal, sig), signal_handler)
async def run(loop):
# Use borrowed connection for NATS then mount NATS Streaming
# client on top.
nc = NATS()
await nc.connect(io_loop=loop, servers=["http://nats-srv:4222"])
# Start session with NATS Streaming cluster.
sc = STAN()
await sc.connect("opuscm", "submittals", nats=nc)
# Synchronous Publisher, does not return until an ack
# has been received from NATS Streaming.
await sc.publish("hi", b'hello')
await sc.publish("hi", b'world')
total_messages = 0
future = asyncio.Future(loop=loop)
async def cb(msg):
nonlocal future
nonlocal total_messages
print("Received a message (seq={}): {}".format(msg.seq, msg.data))
total_messages += 1
# Subscribe to get all messages since beginning.
sub = await sc.subscribe("test", start_at='first', cb=cb)
async def start_agent():
cluster_id = "test-cluster"
client_id = "agent1"
channel_subject = "agent.1AX24"
nats_connection = NATS()
await nats_connection.connect("nats://127.0.0.1:4222")
sc = STAN()
await sc.connect(cluster_id, client_id, nats=nats_connection)
async def cb(msg):
await sc.ack(msg)
req_data = json.loads(msg.data.decode())
global agent_running
if req_data['event_type'] == 'start' and not agent_running:
agent_running = True
config = req_data['config']
print("Agent started with the following configuration: " +
str(config))
# START AGENT CODE
elif req_data['event_type'] == 'stop':
agent_running = False
print("Agent stopped")
# STOP AGENT CODE
await sc.subscribe(channel_subject,
manual_acks=True,
start_at="last_received",
cb=cb)
async def initialize(self, app=None):
# No asyncio loop to run
async with self.lock:
self.nc = NATS()
options = {
"servers": self._hosts,
"loop": self._loop,
"disconnected_cb": self.disconnected_cb,
"reconnected_cb": self.reconnected_cb,
"error_cb": self.error_cb,
"closed_cb": self.closed_cb,
"name": self._name,
"verbose": True,
}
try:
await self.nc.connect(**options)
except ErrNoServers:
logger.exception("No servers found")
raise
logger.info("Connected to nats")
if self._stan is not None:
self.sc = STAN()
await self.sc.connect(
self._stan,
self._uuid,
nats=self.nc,
ping_interval=self._stan_ping_interval,
ping_max_out=self._stan_ping_max_out,
connect_timeout=self._stan_timeout,
)
logger.info("Connected to stan")
self._initialized = True
async def run(loop, msg_id, trigger):
# Use borrowed connection for NATS then mount NATS Streaming
# client on top.
URL = "nats://[email protected]:4222"
clusterID = "fissionMQTrigger"
clientID = "clientPub"
subj = "test1"
nc = NATS()
await nc.connect(io_loop=loop, servers=[URL])
# Start session with NATS Streaming cluster.
sc = STAN()
await sc.connect(clusterID, clientID, nats=nc)
# Synchronous Publisher, does not return until an ack
# has been received from NATS Streaming.
timestamp = time.time()
msg = {'msg_id': msg_id, 'trigger': float(trigger), 'start': timestamp}
encoded = json.dumps(msg).encode('utf-8')
await sc.publish(subj, encoded)
# Close NATS Streaming session
await sc.close()
# We are using a NATS borrowed connection so we need to close manually.
await nc.close()
async def run(loop):
nc = NATS()
sc = STAN()
await nc.connect(io_loop=loop)
await sc.connect("test-cluster", "client-123", max_pub_acks_inflight=512, nats=nc)
acks = []
msgs = []
async def cb(msg):
nonlocal sc
print("Received a message on subscription (seq: {} | recv: {}): {}".format(msg.sequence, len(msgs), msg.data))
msgs.append(msg)
# This will eventually add up causing redelivery to occur.
await asyncio.sleep(0.01, loop=loop)
await sc.ack(msg)
# Use manual acking and have message redelivery be done
# if we do not ack back in 1 second capping to 128 inflight messages.
await sc.subscribe(
"foo", start_at='first', cb=cb, max_inflight=128, manual_acks=True, ack_wait=1)
for i in range(0, 2048):
await sc.publish("foo", b'hello-world')
for i in range(0, 10):
await asyncio.sleep(1, loop=loop)
await sc.close()
await nc.close()
async def run(loop, q):
# Use borrowed connection for NATS then mount NATS Streaming
# client on top.
nc = NATS()
await nc.connect(servers=["nats://nats-streaming:4222"], io_loop=loop)
# Start session with NATS Streaming cluster.
sc = STAN()
await sc.connect("test-cluster", "analytics-client-1", nats=nc)
async def cb(msg):
q.put(msg.data)
logging.info("Received a message (seq={}): {}".format(
msg.seq, msg.data))
# Subscribe to get all messages since beginning.
sub = await sc.subscribe("analytics", start_at='first', cb=cb)
while True:
await asyncio.sleep(1, loop=loop)
await sub.unsubscribe()
# Close NATS Streaming session
await sc.close()
# We are using a NATS borrowed connection so we need to close manually.
await nc.close()
async def run(self, loop):
self.nc = NATS()
await self.nc.connect('localhost:4223', io_loop=loop)
# Start session with NATS Streaming cluster.
self.sc = STAN()
await self.sc.connect("test-cluster", "client-123", nats=self.nc)
async def run(self, loop, handler):
# Use borrowed connection for NATS then mount NATS Streaming
# client on top.
logging.info('Connecting to nats://{0}:{1}...'.format(
self.nats_address, self.nats_port))
nc = NATS()
await nc.connect('{0}:{1}'.format(self.nats_address, self.nats_port),
loop=loop)
# Start session with NATS Streaming cluster.
logging.info(
'Establishing connection to cluster: {0} with clientID: {1}...'.
format(self.cluster_id, self.client_id))
sc = STAN()
await sc.connect(self.cluster_id, self.client_id, nats=nc)
# Send msg.data to handler (KubernetesClient.launch_job())
async def cb(msg):
data = json.loads(msg.data.decode('utf-8'))
handler(data)
try:
await sc.subscribe(self.subject, queue=self.queue_group, cb=cb)
logging.info('Listening on "{0}", queue "{1}"'.format(
self.subject, self.queue_group))
except asyncio.CancelledError as e:
await sc.close()
await nc.close()
raise e
async def run(loop):
nc = NATS()
sc = STAN()
options = {
"servers": ["nats://100.97.218.207:4222"],
"io_loop": loop,
"error_cb": error_cb
}
print("Sever: ", options["servers"])
# variables de Conexion Nats Server
await nc.connect(**options)
# First connect to NATS, then start session with NATS Streaming.
# the established NATS connection.
await sc.connect("vsblty-cluster", Hostname, nats=nc)
# Periodically send a message Cada Segundo
while True:
# Build Json Mensaje
ChannelNats = 'Face-Detection'
data = {}
data['Channel'] = ChannelNats
data['Date'] = str(datetime.now())
data['Frame'] = Image64
data['Hostname'] = Hostname
# Definir Channel y Mensaje
await sc.publish(ChannelNats, bytes(str(data), 'utf8'))
# Sleep
await asyncio.sleep(1, loop=loop)
async def run(loop, ClientId, ServerIP):
nc = NATS()
sc = STAN()
options = {"servers": [ServerIP], "io_loop": loop, "error_cb": error_cb}
await nc.connect(**options)
# Start session with NATS Streaming cluster using
# the established NATS connection.
await sc.connect("vsblty-cluster", ClientId, nats=nc)
async def cb(msg):
# Creando task, llamando a la Funcion FrameProces
task1 = await loop.create_task(FrameProces(msg.seq, msg.data))
#/////////////////////////////////////////////////////////////////////////////
# Publicar Mensaje con el resultado en el canal VSBLTY-DATA-FACE
#/////////////////////////////////////////////////////////////////////////////
ChannelNats = 'VSBLTY-DATA-FACE'
# Definir Channel y Mensaje
await sc.publish(ChannelNats, bytes(str(task1), 'utf8'))
print(
" - [publisher] - Sending Message To Channel 'VSBLTY-DATA-FACE': - "
+ str(msg.seq))
print('')
# Subscribe to get all messages from the beginning.
await sc.subscribe("VSBLTY-DATA-TEMP", start_at='first', cb=cb)
async def set_up(self):
logging.basicConfig(
format="%(asctime)s - %(name)s:%(levelname)s - %(message)s",
datefmt="%d-%b-%y %H:%M:%S",
level=logging.DEBUG,
)
logging.getLogger("asyncio").setLevel(logging.WARNING)
# loop.set_debug(True)
self.nc = NATS()
self.nc._max_payload = 2097152
# logger.debug("in setup")
logger.debug(
f'connect to nats server {self.settings["NOKKHUM_MESSAGE_NATS_HOST"]}'
)
await self.nc.connect(
self.settings["NOKKHUM_MESSAGE_NATS_HOST"],
disconnected_cb=self.disconnected_cb,
)
self.sc = STAN()
await self.sc.connect(
self.settings["NOKKHUM_STAN_CLUSTER"],
"streaming-sub",
nats=self.nc,
)
# logger.debug("connected")
self.running = True
loop = asyncio.get_event_loop()
loop.create_task(self.put_image_to_queue())
async def run(loop):
# Use borrowed connection for NATS then mount NATS Streaming
# client on top.
nc = NATS()
await nc.connect(io_loop=loop)
# Start session with NATS Streaming cluster.
sc = STAN()
await sc.connect("test-cluster", "client-123", nats=nc)
# Synchronous Publisher, does not return until an ack
# has been received from NATS Streaming.
await sc.publish("hi", b'hello')
await sc.publish("hi", b'world')
async def cb(msg):
print("Received a message (seq={}): {}".format(msg.seq, msg.data))
# Subscribe to get all messages since beginning.
sub = await sc.subscribe("hi", start_at='first', cb=cb)
await sub.unsubscribe()
# Close NATS Streaming session
await sc.close()
# We are using a NATS borrowed connection so we need to close manually.
await nc.close()
async def test_async_publish_and_acks(self):
nc = NATS()
await nc.connect(loop=self.loop)
sc = STAN()
await sc.connect("test-cluster", generate_client_id(), nats=nc)
future = asyncio.Future(loop=self.loop)
packs = []
# It will be receiving the ack which we will be controlling manually.
async def cb(ack):
nonlocal packs
nonlocal future
packs.append(ack)
if len(packs) == 1024:
future.set_result(True)
for i in range(0, 1024):
await sc.publish("hi", b'hello', ack_handler=cb)
try:
await asyncio.wait_for(future, 2, loop=self.loop)
except:
pass
# Expect to have received all messages already by now.
self.assertEqual(len(packs), 1024)
# Check that we have cleaned up the pub ack map
self.assertEqual(len(sc._pub_ack_map), 0)
await sc.close()
await nc.close()
self.assertFalse(nc.is_connected)
async def test_distribute_queue_messages(self):
clients = {}
class Component:
def __init__(self, nc, sc):
self.nc = nc
self.sc = sc
self.msgs = []
async def cb(self, msg):
self.msgs.append(msg)
# A...E
for letter in range(ord('A'), ord('F')):
nc = NATS()
sc = STAN()
client_id = "{}-{}".format(generate_client_id(), chr(letter))
await nc.connect(name=client_id, loop=self.loop)
await sc.connect("test-cluster", client_id, nats=nc)
clients[client_id] = Component(nc, sc)
for (client_id, c) in clients.items():
# Messages will be distributed among these subscribers.
await c.sc.subscribe("tasks", queue="group", cb=c.cb)
# Get the first client to publish some messages
acks = []
future = asyncio.Future(loop=self.loop)
async def ack_handler(ack):
nonlocal acks
acks.append(ack)
# Check if all messages have been published already.
if len(acks) >= 2048:
future.set_result(True)
pc = list(clients.values())[-1]
for i in range(0, 2048):
await pc.sc.publish("tasks",
"task-{}".format(i).encode(),
ack_handler=ack_handler)
try:
# Removing the wait here causes the loop to eventually
# stop receiving messages...
await asyncio.wait_for(future, 2, loop=self.loop)
except:
pass
total = 0
for (client_id, c) in clients.items():
# Not perfect but all should have had around
# enough messages...
self.assertTrue(len(c.msgs) > 400)
total += len(c.msgs)
await c.sc.close()
await c.nc.close()
self.assertEqual(total, 2048)
async def test_subscribe_error_callback(self):
client_id = generate_client_id()
with (await nats.connect(loop=self.loop)) as nc:
sc = STAN()
await sc.connect("test-cluster", client_id, nats=nc)
ex = Exception("random error")
error_cb_calls = []
async def cb_foo(msg):
raise ex
async def cb_foo_error(err):
nonlocal error_cb_calls
error_cb_calls.append(err)
sub_foo = await sc.subscribe("foo",
cb=cb_foo,
error_cb=cb_foo_error)
await sc.publish("foo", b"hi")
# Should try to receive the message now
await asyncio.sleep(0.5, loop=self.loop)
# Error callback should have been called once with our exception
self.assertEqual(error_cb_calls, [ex])
await sc.close()
async def test_subscribe_get_pending_queue_size(self):
pmsgs = [] # Plain subscription messages
client_id = generate_client_id()
with (await nats.connect(loop=self.loop)) as nc:
sc = STAN()
await sc.connect("test-cluster", client_id, nats=nc)
# Stagger sending messages and then only retrieve based on timestamp
for i in range(0, 100):
await sc.publish("foo", "hi-{}".format(i).encode())
async def cb_foo(msg):
nonlocal pmsgs
nonlocal sc
await asyncio.sleep(0.2)
await sc.ack(msg)
pmsgs.append(msg)
sub_foo = await sc.subscribe("foo",
deliver_all_available=True,
manual_acks=True,
cb=cb_foo)
self.assertTrue(sub_foo.pending_queue_size > 0)
await asyncio.sleep(1, loop=self.loop)
await sc.close()
async def test_subscribe_with_manual_acks_missing_redelivery(self):
pmsgs = [] # Plain subscription messages
client_id = generate_client_id()
with (await nats.connect(loop=self.loop)) as nc:
sc = STAN()
await sc.connect("test-cluster", client_id, nats=nc)
# Stagger sending messages and then only retrieve based on timestamp
for i in range(0, 100):
await sc.publish("foo", "hi-{}".format(i).encode())
async def cb_foo(msg):
nonlocal pmsgs
nonlocal sc
pmsgs.append(msg)
sub_foo = await sc.subscribe("foo",
ack_wait=1,
deliver_all_available=True,
manual_acks=True,
max_inflight=1,
cb=cb_foo)
# Should try to receive all the messages now
await asyncio.sleep(2.5, loop=self.loop)
# Only get one since not acking back, will get same message a few times.
self.assertEqual(len(pmsgs), 3)
await sc.close()
def __init__(self,
host='127.0.0.1',
port='4222',
streaming_id=None,
cluster='svom-cluster'):
self.nc = NATS()
self.server = host
self.is_connected = False
self.streaming_id = streaming_id
self.cluster = cluster
self.sc = None
self.loop = None
try:
self.loop = asyncio.get_event_loop()
except RuntimeError:
LOG.info(
'Failed to retrieve asyncio event loop. Creating a new one...')
self.loop = asyncio.new_event_loop()
self.loop.run_until_complete(self._connect(servers=[self.server]))
if streaming_id is not None:
if has_streaming is True:
self.sc = STAN()
self.loop.run_until_complete(self._stan_connect())
else:
LOG.error("Can't run client without asyncio-nats-streaming")
sys.exit(1)
self.nats_server = NATSClient(self.nc, self.loop)
self.nats_server.start()
async def run(loop, result):
# Use borrowed connection for NATS then mount NATS Streaming
# client on top.
nc = NATS()
await nc.connect(
nats_hostport,
io_loop=loop,
max_reconnect_attempts=10,
reconnected_cb=reconnected_cb,
disconnected_cb=disconnected_cb,
)
# Start session with NATS Streaming cluster.
sc = STAN()
await sc.connect("test-cluster", "client-pub", nats=nc)
print(result)
r = json.dumps(result).encode('utf-8')
print(r)
print(type(r))
await sc.publish("hi", r)
#await sc.publish("hi", b'world')
#await asyncio.sleep(1, loop=loop)
await sc.close()
await nc.close()
async def run(loop):
nc = NATS()
sc = STAN()
await nc.connect(io_loop=loop)
await sc.connect("test-cluster", "client-123", nats=nc)
async def ack_handler(ack):
print("Received ack: {}".format(ack.guid))
for i in range(0, 10):
await sc.publish("foo", b'hello-world', ack_handler=ack_handler)
async def cb(msg):
nonlocal sc
print("Received a message on subscription (seq: {}): {}".format(
msg.sequence, msg.data))
await sc.ack(msg)
# Use manual acking and have message redelivery be done
# if we do not ack back in 1 second.
await sc.subscribe("foo",
start_at='first',
cb=cb,
manual_acks=True,
ack_wait=1)
for i in range(0, 5):
await asyncio.sleep(1, loop=loop)
await sc.close()
await nc.close()
async def connect(self):
self.__nc = NATS()
await self.__nc.connect(io_loop=self.__loop,
servers=["http://nats-srv:4222"])
self.__sc = STAN()
await self.__sc.connect("opuscm", "submittals", nats=self.__nc)
print("Connected to NATS")
async def _run(self, loop, event_callback, execution_callback):
print("starting client StreamService...")
nc = NATS()
sc = STAN()
async def disconnected_cb():
print("Disconnected from StreamService...")
async def reconnected_cb():
print("Reconnected to StreamService...")
await nc.connect("host.docker.internal:4223",
reconnected_cb=reconnected_cb,
disconnected_cb=disconnected_cb,
max_reconnect_attempts=-1,
reconnect_time_wait=10,
loop=loop)
# demo client id. Each client would have a separate id ideally
await sc.connect("test-cluster", "client-id-123", nats=nc)
# example of potential subscription feed names so wildcards could be used in the future
events_subject = "feed.events"
executions_subject = "feed.executions"
# last_received ensures a client won't receive stale messages and starts with
# most recently published value
await sc.subscribe(events_subject,
start_at="last_received",
cb=event_callback)
await sc.subscribe(executions_subject,
start_at="last_received",
cb=execution_callback)
async def publish(payload, subject: str, client_name: str = None): # pylint: disable=too-few-public-methods
"""Service to manage Queue publish operations."""
current_app.logger.debug('<publish')
# NATS client connections
nats_con = NATS()
stan_con = STAN()
async def close():
"""Close the stream and nats connections."""
await stan_con.close()
await nats_con.close()
try:
current_app.logger.info('Publishing to %s', subject)
# Connect to the NATS server, and then use that for the streaming connection.
await nats_con.connect(**nats_connection_options(
client_name=client_name),
verbose=True,
connect_timeout=3,
reconnect_time_wait=1)
await stan_con.connect(**stan_connection_options(nats_con=nats_con))
current_app.logger.debug(payload)
await stan_con.publish(subject=subject,
payload=json.dumps(payload).encode('utf-8'))
except Exception as e: # NOQA # pylint: disable=broad-except
current_app.logger.error(e)
raise
finally:
# await nc.flush()
await close()
current_app.logger.debug('>publish')
async def _connect(self):
self.nc = NATS()
await self.nc.connect(io_loop=self.loop)
self.sc = STAN()
await self.sc.connect(self.cluster_id,
self.client_name,
nats=self.nc)
async def run(self, loop):
self.nc = NATS()
await self.nc.connect('localhost:4222', io_loop=loop)
# Start session with NATS Streaming cluster.
self.sc = STAN()
await self.sc.connect("test-cluster", "client-456", nats=self.nc)
await self.sc.subscribe("nokkhum.streaming.processors", cb=self.handle_msg)