class NsqPublisher(
AbstractPublisher, ):
"""
使用nsq作为中间件
"""
# noinspection PyAttributeOutsideInit
def custom_init(self):
self._nsqd_cleint = NsqdHTTPClient(frame_config.NSQD_HTTP_CLIENT_HOST,
frame_config.NSQD_HTTP_CLIENT_PORT)
self._producer = Producer(frame_config.NSQD_TCP_ADDRESSES)
self._producer.start()
def concrete_realization_of_publish(self, msg):
# noinspection PyTypeChecker
self._producer.publish(self._queue_name, msg.encode())
def clear(self):
try:
self._nsqd_cleint.empty_topic(self._queue_name)
except NSQHttpError as e:
self.logger.exception(e) # 不能清除一个不存在的topoc会报错，和其他消息队列中间件不同。
self.logger.warning(f'清除 {self._queue_name} topic中的消息成功')
def get_message_count(self):
return self._nsqd_cleint.stats(
self._queue_name)['topics'][0]['message_count']
def close(self):
self._producer.close()
def test_tls_publish():
extra_params = [
'--tls-required', 'true',
'--https-address', '127.0.0.1:4152',
]
with NsqdIntegrationServer(extra_params=extra_params) as server:
producer = Producer(
server.tcp_address,
tls_options={
'keyfile': server.tls_key,
'certfile': server.tls_cert,
},
tls_v1=True,
)
producer.start()
for _ in range(100):
producer.publish('test', b'hi')
producer.close()
producer.join()
conn = NsqdHTTPClient(
server.address,
'4152',
connection_class=urllib3.HTTPSConnectionPool,
cert_reqs='CERT_NONE',
)
stats = conn.stats()
assert stats['topics'][0]['depth'] == 100
def test_max_concurrency():
server1 = NsqdIntegrationServer()
server2 = NsqdIntegrationServer()
with server1, server2:
class Accounting(object):
count = 0
total = 100
concurrency = 0
error = None
for server in (server1, server2):
conn = NsqdHTTPClient(server.address, server.http_port)
for _ in range(Accounting.total // 2):
conn.publish('test', b'danger zone!')
reader = Reader(
topic='test',
channel='test',
nsqd_tcp_addresses=[
server1.tcp_address,
server2.tcp_address,
],
max_in_flight=5,
max_concurrency=1,
)
@reader.on_exception.connect
def error_handler(reader, message, error):
if isinstance(error, NSQSocketError):
return
Accounting.error = error
reader.close()
@reader.on_message.connect
def handler(reader, message):
assert message.body == b'danger zone!'
assert Accounting.concurrency == 0
Accounting.concurrency += 1
gevent.sleep()
Accounting.concurrency -= 1
Accounting.count += 1
if Accounting.count == Accounting.total:
reader.close()
reader.start()
if Accounting.error:
raise Accounting.error
assert Accounting.count == Accounting.total
def test_backoff():
with NsqdIntegrationServer() as server:
conn = NsqdHTTPClient(server.address, server.http_port)
for _ in range(500):
conn.publish('test', 'danger zone!')
consumer = Consumer(
topic='test',
channel='test',
nsqd_tcp_addresses=[server.tcp_address],
max_in_flight=100,
message_handler=lambda consumer, message: None
)
consumer.start(block=False)
consumer._redistributed_ready_event.wait()
conn = next(iter(consumer._connections))
consumer._message_backoffs[conn].failure()
consumer._message_backoffs[conn].failure()
consumer._start_backoff(conn)
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.BACKOFF
assert consumer.total_ready_count == 0
consumer._start_throttled(conn)
consumer._redistribute_ready_state()
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.THROTTLED
assert consumer.total_ready_count == 1
consumer._message_backoffs[conn].success()
consumer._complete_backoff(conn)
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.BACKOFF
assert consumer.total_ready_count == 0
consumer._start_throttled(conn)
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.THROTTLED
assert consumer.total_ready_count == 1
consumer._message_backoffs[conn].success()
consumer._complete_backoff(conn)
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.RUNNING
assert consumer.total_ready_count == 100
def test_backoff():
with NsqdIntegrationServer() as server:
conn = NsqdHTTPClient(server.address, server.http_port)
for _ in range(500):
conn.publish('test', 'danger zone!')
consumer = Consumer(topic='test',
channel='test',
nsqd_tcp_addresses=[server.tcp_address],
max_in_flight=100,
message_handler=lambda consumer, message: None)
consumer.start(block=False)
consumer._redistributed_ready_event.wait()
conn = next(iter(consumer._connections))
consumer._message_backoffs[conn].failure()
consumer._message_backoffs[conn].failure()
consumer._start_backoff(conn)
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.BACKOFF
assert consumer.total_ready_count == 0
consumer._start_throttled(conn)
consumer._redistribute_ready_state()
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.THROTTLED
assert consumer.total_ready_count == 1
consumer._message_backoffs[conn].success()
consumer._complete_backoff(conn)
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.BACKOFF
assert consumer.total_ready_count == 0
consumer._start_throttled(conn)
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.THROTTLED
assert consumer.total_ready_count == 1
consumer._message_backoffs[conn].success()
consumer._complete_backoff(conn)
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.RUNNING
assert consumer.total_ready_count == 100
def test_multipublish():
with NsqdIntegrationServer() as server:
producer = Producer(server.tcp_address)
producer.start()
for _ in range(10):
producer.multipublish('test', 10 * [b'hi'])
producer.close()
producer.join()
conn = NsqdHTTPClient(server.address, server.http_port)
stats = conn.stats()
assert stats['topics'][0]['depth'] == 100
def test_multipublish():
with NsqdIntegrationServer() as server:
producer = Producer(server.tcp_address)
producer.start()
for _ in range(10):
producer.multipublish('test', 10 * [b'hi'])
producer.close()
producer.join()
conn = NsqdHTTPClient(server.address, server.http_port)
stats = conn.stats()
assert stats['topics'][0]['depth'] == 100
def test_lookupd():
with LookupdIntegrationServer() as lookupd_server:
server1 = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address)
server2 = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address)
with server1, server2:
class Accounting(object):
count = 0
total = 500
concurrency = 0
error = None
for server in (server1, server2):
conn = NsqdHTTPClient(server.address, server.http_port)
for _ in range(Accounting.total // 2):
conn.publish('test', b'danger zone!')
consumer = Consumer(
topic='test',
channel='test',
lookupd_http_addresses=lookupd_server.http_address,
max_in_flight=32,
)
@consumer.on_exception.connect
def error_handler(consumer, message, error):
if isinstance(error, NSQSocketError):
return
Accounting.error = error
consumer.close()
@consumer.on_message.connect
def handler(consumer, message):
assert message.body == b'danger zone!'
Accounting.count += 1
if Accounting.count == Accounting.total:
consumer.close()
gevent.sleep(0.1)
consumer.start()
if Accounting.error:
raise Accounting.error
assert Accounting.count == Accounting.total
def test_lookupd():
with LookupdIntegrationServer() as lookupd_server:
server1 = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address)
server2 = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address)
with server1, server2:
class Accounting(object):
count = 0
total = 500
concurrency = 0
error = None
for server in (server1, server2):
conn = NsqdHTTPClient(server.address, server.http_port)
for _ in range(Accounting.total // 2):
conn.publish('test', b'danger zone!')
consumer = Consumer(
topic='test',
channel='test',
lookupd_http_addresses=lookupd_server.http_address,
max_in_flight=32,
)
@consumer.on_exception.connect
def error_handler(consumer, message, error):
if isinstance(error, NSQSocketError):
return
Accounting.error = error
consumer.close()
@consumer.on_message.connect
def handler(consumer, message):
assert message.body == b'danger zone!'
Accounting.count += 1
if Accounting.count == Accounting.total:
consumer.close()
gevent.sleep(0.1)
consumer.start()
if Accounting.error:
raise Accounting.error
assert Accounting.count == Accounting.total
def test_async_publish():
with NsqdIntegrationServer() as server:
results = []
producer = Producer(server.tcp_address)
producer.start()
for _ in range(100):
results.append(producer.publish('test', b'hi', raise_error=False))
gevent.joinall(results, raise_error=True)
producer.close()
producer.join()
conn = NsqdHTTPClient(server.address, server.http_port)
stats = conn.stats()
assert stats['topics'][0]['depth'] == 100
def test_async_publish():
with NsqdIntegrationServer() as server:
results = []
producer = Producer(server.tcp_address)
producer.start()
for _ in range(100):
results.append(producer.publish('test', b'hi', raise_error=False))
gevent.joinall(results, raise_error=True)
producer.close()
producer.join()
conn = NsqdHTTPClient(server.address, server.http_port)
stats = conn.stats()
assert stats['topics'][0]['depth'] == 100
def test_messages():
with NsqdIntegrationServer() as server:
class Accounting(object):
count = 0
total = 500
error = None
conn = NsqdHTTPClient(server.address, server.http_port)
for _ in range(Accounting.total):
conn.publish('test', b'danger zone!')
consumer = Consumer(
topic='test',
channel='test',
nsqd_tcp_addresses=[server.tcp_address],
max_in_flight=100,
)
@consumer.on_exception.connect
def error_handler(consumer, message, error):
if isinstance(error, NSQSocketError):
return
Accounting.error = error
consumer.close()
@consumer.on_message.connect
def handler(consumer, message):
assert message.body == b'danger zone!'
Accounting.count += 1
if Accounting.count == Accounting.total:
consumer.close()
consumer.start()
if Accounting.error:
raise Accounting.error
assert Accounting.count == Accounting.total
def test_messages():
with NsqdIntegrationServer() as server:
class Accounting(object):
count = 0
total = 500
error = None
conn = NsqdHTTPClient(server.address, server.http_port)
for _ in range(Accounting.total):
conn.publish('test', b'danger zone!')
consumer = Consumer(
topic='test',
channel='test',
nsqd_tcp_addresses=[server.tcp_address],
max_in_flight=100,
)
@consumer.on_exception.connect
def error_handler(consumer, message, error):
if isinstance(error, NSQSocketError):
return
Accounting.error = error
consumer.close()
@consumer.on_message.connect
def handler(consumer, message):
assert message.body == b'danger zone!'
Accounting.count += 1
if Accounting.count == Accounting.total:
consumer.close()
consumer.start()
if Accounting.error:
raise Accounting.error
assert Accounting.count == Accounting.total
def custom_init(self):
self._nsqd_cleint = NsqdHTTPClient(frame_config.NSQD_HTTP_CLIENT_HOST,
frame_config.NSQD_HTTP_CLIENT_PORT)
self._producer = Producer(frame_config.NSQD_TCP_ADDRESSES)
self._producer.start()