def test_lookup():
with LookupdIntegrationServer() as lookupd_server:
nsqd_server = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address)
with NsqdIntegrationServer(
lookupd=lookupd_server.tcp_address) as nsqd_server:
lookupd = gnsq.LookupdClient(lookupd_server.address,
lookupd_server.http_port)
nsqd = gnsq.NsqdHTTPClient(nsqd_server.address,
nsqd_server.http_port)
gevent.sleep(0.1)
assert len(lookupd.topics()['topics']) == 0
assert len(lookupd.channels('topic')['channels']) == 0
assert len(lookupd.nodes()['producers']) == 1
nsqd.create_topic('topic')
gevent.sleep(0.1)
info = lookupd.lookup('topic')
assert len(info['channels']) == 0
assert len(info['producers']) == 1
assert len(lookupd.topics()['topics']) == 1
assert len(lookupd.channels('topic')['channels']) == 0
nsqd.create_channel('topic', 'channel')
gevent.sleep(0.1)
info = lookupd.lookup('topic')
assert len(info['channels']) == 1
assert len(info['producers']) == 1
assert len(lookupd.topics()['topics']) == 1
assert len(lookupd.channels('topic')['channels']) == 1
def test_max_concurrency():
server1 = NsqdIntegrationServer()
server2 = NsqdIntegrationServer()
with server1, server2:
class Accounting(object):
count = 0
total = 100
concurrency = 0
error = None
for server in (server1, server2):
conn = Nsqd(
address=server.address,
tcp_port=server.tcp_port,
http_port=server.http_port,
)
for _ in range(Accounting.total // 2):
conn.publish_http('test', b'danger zone!')
reader = Reader(
topic='test',
channel='test',
nsqd_tcp_addresses=[
server1.tcp_address,
server2.tcp_address,
],
max_in_flight=5,
max_concurrency=1,
)
@reader.on_exception.connect
def error_handler(reader, message, error):
if isinstance(error, NSQSocketError):
return
Accounting.error = error
reader.close()
@reader.on_message.connect
def handler(reader, message):
assert message.body == b'danger zone!'
assert Accounting.concurrency == 0
Accounting.concurrency += 1
gevent.sleep()
Accounting.concurrency -= 1
Accounting.count += 1
if Accounting.count == Accounting.total:
reader.close()
reader.start()
if Accounting.error:
raise Accounting.error
assert Accounting.count == Accounting.total
def test_lookupd():
with LookupdIntegrationServer() as lookupd_server:
server1 = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address)
server2 = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address)
with server1, server2:
class Accounting(object):
count = 0
total = 500
concurrency = 0
error = None
for server in (server1, server2):
conn = Nsqd(
address=server.address,
tcp_port=server.tcp_port,
http_port=server.http_port,
)
for _ in range(Accounting.total // 2):
conn.publish_http('test', b'danger zone!')
reader = Reader(
topic='test',
channel='test',
lookupd_http_addresses=lookupd_server.http_address,
max_in_flight=32,
)
@reader.on_exception.connect
def error_handler(reader, message, error):
if isinstance(error, NSQSocketError):
return
Accounting.error = error
reader.close()
@reader.on_message.connect
def handler(reader, message):
assert message.body == b'danger zone!'
Accounting.count += 1
if Accounting.count == Accounting.total:
reader.close()
gevent.sleep(0.1)
reader.start()
if Accounting.error:
raise Accounting.error
assert Accounting.count == Accounting.total
def test_topics_channels():
with NsqdIntegrationServer() as server:
conn = gnsq.NsqdHTTPClient(server.address, server.http_port)
assert len(conn.stats()['topics']) == 0
with pytest.raises(gnsq.errors.NSQHttpError):
conn.delete_topic('topic')
conn.create_topic('topic')
topics = conn.stats()['topics']
assert len(topics) == 1
assert topics[0]['topic_name'] == 'topic'
conn.delete_topic('topic')
assert len(conn.stats()['topics']) == 0
with pytest.raises(gnsq.errors.NSQHttpError):
conn.create_channel('topic', 'channel')
with pytest.raises(gnsq.errors.NSQHttpError):
conn.delete_channel('topic', 'channel')
conn.create_topic('topic')
assert len(conn.stats()['topics'][0]['channels']) == 0
conn.create_channel('topic', 'channel')
channels = conn.stats()['topics'][0]['channels']
assert len(channels) == 1
assert channels[0]['channel_name'] == 'channel'
conn.delete_channel('topic', 'channel')
assert len(conn.stats()['topics'][0]['channels']) == 0
def test_backoff():
with NsqdIntegrationServer() as server:
conn = Nsqd(
address=server.address,
tcp_port=server.tcp_port,
http_port=server.http_port,
)
for _ in range(500):
conn.publish_http('test', 'danger zone!')
reader = Reader(topic='test',
channel='test',
nsqd_tcp_addresses=[server.tcp_address],
max_in_flight=100,
message_handler=lambda reader, message: None)
reader.start(block=False)
reader.start_backoff()
assert reader.state == states.THROTTLED
assert reader.total_in_flight_or_ready <= 1
reader.complete_backoff()
assert reader.state == states.RUNNING
def test_lookup():
lookupd_server = LookupdIntegrationServer()
nsqd_server = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address)
@with_all(lookupd_server, nsqd_server)
def _(lookupd_server, nsqd_server):
lookupd = gnsq.Lookupd(lookupd_server.http_address)
conn = gnsq.Nsqd(nsqd_server.address, http_port=nsqd_server.http_port)
assert len(lookupd.topics()['topics']) == 0
assert len(lookupd.channels('topic')['channels']) == 0
assert len(lookupd.nodes()['producers']) == 1
conn.create_topic('topic')
gevent.sleep(0.1)
info = lookupd.lookup('topic')
assert len(info['channels']) == 0
assert len(info['producers']) == 1
assert len(lookupd.topics()['topics']) == 1
assert len(lookupd.channels('topic')['channels']) == 0
conn.create_channel('topic', 'channel')
gevent.sleep(0.1)
info = lookupd.lookup('topic')
assert len(info['channels']) == 1
assert len(info['producers']) == 1
assert len(lookupd.topics()['topics']) == 1
assert len(lookupd.channels('topic')['channels']) == 1
def test_publish():
with NsqdIntegrationServer() as server:
conn = gnsq.NsqdHTTPClient(server.address, server.http_port)
conn.publish('topic', b'sup')
assert conn.stats()['topics'][0]['depth'] == 1
conn.multipublish('topic', [b'sup', b'sup'])
assert conn.stats()['topics'][0]['depth'] == 3
conn.multipublish('topic', iter([b'sup', b'sup', b'sup']))
assert conn.stats()['topics'][0]['depth'] == 6
conn.empty_topic('topic')
assert conn.stats()['topics'][0]['depth'] == 0
conn.create_topic('topic')
conn.create_channel('topic', 'channel')
conn.publish('topic', b'sup')
assert conn.stats()['topics'][0]['channels'][0]['depth'] == 1
conn.empty_channel('topic', 'channel')
assert conn.stats()['topics'][0]['channels'][0]['depth'] == 0
if server.version < (0, 3, 6):
return
conn.publish('topic', b'sup', 60 * 1000)
stats = conn.stats()
assert stats['topics'][0]['channels'][0]['depth'] == 0
assert stats['topics'][0]['channels'][0]['deferred_count'] == 1
def test_tls_publish():
extra_params = [
'--tls-required', 'true',
'--https-address', '127.0.0.1:4152',
]
with NsqdIntegrationServer(extra_params=extra_params) as server:
producer = Producer(
server.tcp_address,
tls_options={
'keyfile': server.tls_key,
'certfile': server.tls_cert,
},
tls_v1=True,
)
producer.start()
for _ in range(100):
producer.publish('test', b'hi')
producer.close()
producer.join()
conn = NsqdHTTPClient(
server.address,
'4152',
connection_class=urllib3.HTTPSConnectionPool,
cert_reqs='CERT_NONE',
)
stats = conn.stats()
assert stats['topics'][0]['depth'] == 100
def test_publish_error():
with NsqdIntegrationServer() as server:
producer = Producer(server.tcp_address)
producer.start()
with pytest.raises(NSQInvalid):
producer.publish('test', b'hi', defer=-1000)
producer.close()
producer.join()
def test_cls_error():
with NsqdIntegrationServer() as server:
conn = Nsqd(address=server.address, tcp_port=server.tcp_port)
conn.connect()
assert conn.state == states.CONNECTED
conn.close()
frame, error = conn.read_response()
assert frame == nsq.FRAME_TYPE_ERROR
assert isinstance(error, errors.NSQInvalid)
def test_backoff():
with NsqdIntegrationServer() as server:
conn = NsqdHTTPClient(server.address, server.http_port)
for _ in range(500):
conn.publish('test', 'danger zone!')
consumer = Consumer(topic='test',
channel='test',
nsqd_tcp_addresses=[server.tcp_address],
max_in_flight=100,
message_handler=lambda consumer, message: None)
consumer.start(block=False)
consumer._redistributed_ready_event.wait()
conn = next(iter(consumer._connections))
consumer._message_backoffs[conn].failure()
consumer._message_backoffs[conn].failure()
consumer._start_backoff(conn)
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.BACKOFF
assert consumer.total_ready_count == 0
consumer._start_throttled(conn)
consumer._redistribute_ready_state()
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.THROTTLED
assert consumer.total_ready_count == 1
consumer._message_backoffs[conn].success()
consumer._complete_backoff(conn)
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.BACKOFF
assert consumer.total_ready_count == 0
consumer._start_throttled(conn)
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.THROTTLED
assert consumer.total_ready_count == 1
consumer._message_backoffs[conn].success()
consumer._complete_backoff(conn)
consumer._redistribute_ready_state()
assert consumer._connections[conn] == states.RUNNING
assert consumer.total_ready_count == 100
def test_multipublish():
with NsqdIntegrationServer() as server:
producer = Producer(server.tcp_address)
producer.start()
for _ in range(10):
producer.multipublish('test', 10 * [b'hi'])
producer.close()
producer.join()
conn = NsqdHTTPClient(server.address, server.http_port)
stats = conn.stats()
assert stats['topics'][0]['depth'] == 100
def test_messages():
with NsqdIntegrationServer() as server:
class Accounting(object):
count = 0
total = 500
error = None
conn = Nsqd(
address=server.address,
tcp_port=server.tcp_port,
http_port=server.http_port,
)
for _ in range(Accounting.total):
conn.publish_http('test', b'danger zone!')
reader = Reader(
topic='test',
channel='test',
nsqd_tcp_addresses=[server.tcp_address],
max_in_flight=100,
)
@reader.on_exception.connect
def error_handler(reader, message, error):
if isinstance(error, NSQSocketError):
return
Accounting.error = error
reader.close()
@reader.on_message.connect
def handler(reader, message):
assert message.body == b'danger zone!'
Accounting.count += 1
if Accounting.count == Accounting.total:
assert not reader.is_starved
reader.close()
reader.start()
if Accounting.error:
raise Accounting.error
assert Accounting.count == Accounting.total
def test_async_publish():
with NsqdIntegrationServer() as server:
results = []
producer = Producer(server.tcp_address)
producer.start()
for _ in range(100):
results.append(producer.publish('test', b'hi', raise_error=False))
gevent.joinall(results, raise_error=True)
producer.close()
producer.join()
conn = NsqdHTTPClient(server.address, server.http_port)
stats = conn.stats()
assert stats['topics'][0]['depth'] == 100
def test_publish():
with NsqdIntegrationServer() as server:
conn = gnsq.Nsqd(server.address, http_port=server.http_port)
conn.publish('topic', b'sup')
assert conn.stats()['topics'][0]['depth'] == 1
conn.multipublish('topic', ['sup', 'sup'])
assert conn.stats()['topics'][0]['depth'] == 3
conn.multipublish('topic', iter(['sup', 'sup', 'sup']))
assert conn.stats()['topics'][0]['depth'] == 6
conn.empty_topic('topic')
assert conn.stats()['topics'][0]['depth'] == 0
conn.create_topic('topic')
conn.create_channel('topic', 'channel')
conn.publish('topic', b'sup')
assert conn.stats()['topics'][0]['channels'][0]['depth'] == 1
conn.empty_channel('topic', 'channel')
assert conn.stats()['topics'][0]['channels'][0]['depth'] == 0
def test_socket_upgrades(tls, deflate, snappy):
with NsqdIntegrationServer() as server:
options = {
'address': server.address,
'tcp_port': server.tcp_port,
'deflate': deflate,
'snappy': snappy,
}
if tls:
options.update({
'tls_v1': True,
'tls_options': {
'keyfile': server.tls_key,
'certfile': server.tls_cert,
}
})
conn = Nsqd(**options)
conn.connect()
assert conn.state == states.CONNECTED
if deflate and snappy:
with pytest.raises(errors.NSQErrorCode):
conn.identify()
return
if tls and BAD_GEVENT:
with pytest.raises(AttributeError):
conn.identify()
return
if tls and server.version < (0, 2, 28):
with pytest.raises(ssl.SSLError):
conn.identify()
return
resp = conn.identify()
assert isinstance(resp, dict)
assert resp['tls_v1'] is tls
assert resp['deflate'] is deflate
assert resp['snappy'] is snappy
if tls and (deflate or snappy):
assert isinstance(conn.stream.socket._socket, SSLSocket)
elif tls:
assert isinstance(conn.stream.socket, SSLSocket)
if deflate:
assert isinstance(conn.stream.socket, DefalteSocket)
if snappy:
assert isinstance(conn.stream.socket, SnappySocket)
conn.publish('topic', b'sup')
frame, data = conn.read_response()
assert frame == nsq.FRAME_TYPE_RESPONSE
assert data == b'OK'
conn.subscribe('topic', 'channel')
frame, data = conn.read_response()
assert frame == nsq.FRAME_TYPE_RESPONSE
assert data == b'OK'
conn.ready(1)
frame, data = conn.read_response()
assert frame == nsq.FRAME_TYPE_MESSAGE
assert data.body == b'sup'
conn.close_stream()
def test_basic():
with NsqdIntegrationServer() as server:
conn = gnsq.NsqdHTTPClient(server.address, server.http_port)
assert conn.ping() == b'OK'
assert 'topics' in conn.stats()
assert 'version' in conn.info()