def test_backoff():
with NsqdIntegrationServer() as server:
conn = Nsqd(
address=server.address,
tcp_port=server.tcp_port,
http_port=server.http_port,
)
for _ in range(500):
conn.publish_http('test', 'danger zone!')
reader = Reader(
topic='test',
channel='test',
nsqd_tcp_addresses=[server.tcp_address],
max_in_flight=100,
message_handler=lambda reader, message: None
)
reader.start(block=False)
reader.start_backoff()
assert reader.state == states.THROTTLED
assert reader.total_in_flight_or_ready <= 1
reader.complete_backoff()
assert reader.state == states.RUNNING
def test_backoff():
with NsqdIntegrationServer() as server:
conn = Nsqd(
address=server.address,
tcp_port=server.tcp_port,
http_port=server.http_port,
)
for _ in range(500):
conn.publish_http('test', 'danger zone!')
reader = Reader(topic='test',
channel='test',
nsqd_tcp_addresses=[server.tcp_address],
max_in_flight=100,
message_handler=lambda reader, message: None)
reader.start(block=False)
reader.start_backoff()
assert reader.state == states.THROTTLED
assert reader.total_in_flight_or_ready <= 1
reader.complete_backoff()
assert reader.state == states.RUNNING
def test_max_concurrency():
server1 = NsqdIntegrationServer()
server2 = NsqdIntegrationServer()
with server1, server2:
class Accounting(object):
count = 0
total = 100
concurrency = 0
error = None
for server in (server1, server2):
conn = Nsqd(
address=server.address,
tcp_port=server.tcp_port,
http_port=server.http_port,
)
for _ in range(Accounting.total // 2):
conn.publish_http('test', b'danger zone!')
reader = Reader(
topic='test',
channel='test',
nsqd_tcp_addresses=[
server1.tcp_address,
server2.tcp_address,
],
max_in_flight=5,
max_concurrency=1,
)
@reader.on_exception.connect
def error_handler(reader, message, error):
if isinstance(error, NSQSocketError):
return
Accounting.error = error
reader.close()
@reader.on_message.connect
def handler(reader, message):
assert message.body == b'danger zone!'
assert Accounting.concurrency == 0
Accounting.concurrency += 1
gevent.sleep()
Accounting.concurrency -= 1
Accounting.count += 1
if Accounting.count == Accounting.total:
reader.close()
reader.start()
if Accounting.error:
raise Accounting.error
assert Accounting.count == Accounting.total
def _(server1, server2):
class Accounting(object):
count = 0
total = 100
concurrency = 0
error = None
for server in (server1, server2):
conn = Nsqd(
address=server.address,
tcp_port=server.tcp_port,
http_port=server.http_port,
)
for _ in xrange(Accounting.total / 2):
conn.publish_http('test', 'danger zone!')
reader = Reader(
topic='test',
channel='test',
nsqd_tcp_addresses=[
server1.tcp_address,
server2.tcp_address,
],
max_in_flight=5,
max_concurrency=1,
)
@reader.on_exception.connect
def error_handler(reader, message, error):
if isinstance(error, NSQSocketError):
return
Accounting.error = error
reader.close()
@reader.on_message.connect
def handler(reader, message):
assert message.body == 'danger zone!'
assert Accounting.concurrency == 0
Accounting.concurrency += 1
gevent.sleep()
Accounting.concurrency -= 1
Accounting.count += 1
if Accounting.count == Accounting.total:
reader.close()
try:
reader.start()
except NSQSocketError:
pass
if Accounting.error:
raise Accounting.error
assert Accounting.count == Accounting.total
def test_lookupd():
with LookupdIntegrationServer() as lookupd_server:
server1 = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address)
server2 = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address)
with server1, server2:
class Accounting(object):
count = 0
total = 500
concurrency = 0
error = None
for server in (server1, server2):
conn = Nsqd(
address=server.address,
tcp_port=server.tcp_port,
http_port=server.http_port,
)
for _ in range(Accounting.total // 2):
conn.publish_http('test', b'danger zone!')
reader = Reader(
topic='test',
channel='test',
lookupd_http_addresses=lookupd_server.http_address,
max_in_flight=32,
)
@reader.on_exception.connect
def error_handler(reader, message, error):
if isinstance(error, NSQSocketError):
return
Accounting.error = error
reader.close()
@reader.on_message.connect
def handler(reader, message):
assert message.body == b'danger zone!'
Accounting.count += 1
if Accounting.count == Accounting.total:
reader.close()
gevent.sleep(0.1)
reader.start()
if Accounting.error:
raise Accounting.error
assert Accounting.count == Accounting.total
def test_messages():
with NsqdIntegrationServer() as server:
class Accounting(object):
count = 0
total = 500
error = None
conn = Nsqd(
address=server.address,
tcp_port=server.tcp_port,
http_port=server.http_port,
)
for _ in xrange(Accounting.total):
conn.publish_http('test', 'danger zone!')
reader = Reader(
topic='test',
channel='test',
nsqd_tcp_addresses=[server.tcp_address],
max_in_flight=100,
)
@reader.on_exception.connect
def error_handler(reader, message, error):
if isinstance(error, NSQSocketError):
return
Accounting.error = error
reader.close()
@reader.on_message.connect
def handler(reader, message):
assert message.body == 'danger zone!'
Accounting.count += 1
if Accounting.count == Accounting.total:
assert not reader.is_starved
reader.close()
try:
reader.start()
except NSQSocketError:
pass
if Accounting.error:
raise Accounting.error
assert Accounting.count == Accounting.total
def test_messages():
with NsqdIntegrationServer() as server:
class Accounting(object):
count = 0
total = 500
error = None
conn = Nsqd(
address=server.address,
tcp_port=server.tcp_port,
http_port=server.http_port,
)
for _ in range(Accounting.total):
conn.publish_http('test', b'danger zone!')
reader = Reader(
topic='test',
channel='test',
nsqd_tcp_addresses=[server.tcp_address],
max_in_flight=100,
)
@reader.on_exception.connect
def error_handler(reader, message, error):
if isinstance(error, NSQSocketError):
return
Accounting.error = error
reader.close()
@reader.on_message.connect
def handler(reader, message):
assert message.body == b'danger zone!'
Accounting.count += 1
if Accounting.count == Accounting.total:
assert not reader.is_starved
reader.close()
reader.start()
if Accounting.error:
raise Accounting.error
assert Accounting.count == Accounting.total
def _(server1, server2):
class Accounting(object):
count = 0
total = 500
concurrency = 0
error = None
for server in (server1, server2):
conn = Nsqd(
address=server.address,
tcp_port=server.tcp_port,
http_port=server.http_port,
)
for _ in range(Accounting.total // 2):
conn.publish_http('test', b'danger zone!')
reader = Reader(
topic='test',
channel='test',
lookupd_http_addresses=lookupd_server.http_address,
max_in_flight=32,
)
@reader.on_exception.connect
def error_handler(reader, message, error):
if isinstance(error, NSQSocketError):
return
Accounting.error = error
reader.close()
@reader.on_message.connect
def handler(reader, message):
assert message.body == b'danger zone!'
Accounting.count += 1
if Accounting.count == Accounting.total:
reader.close()
gevent.sleep(0.1)
reader.start()
if Accounting.error:
raise Accounting.error
assert Accounting.count == Accounting.total
