def test_lookup(): with LookupdIntegrationServer() as lookupd_server: nsqd_server = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address) with NsqdIntegrationServer( lookupd=lookupd_server.tcp_address) as nsqd_server: lookupd = gnsq.LookupdClient(lookupd_server.address, lookupd_server.http_port) nsqd = gnsq.NsqdHTTPClient(nsqd_server.address, nsqd_server.http_port) gevent.sleep(0.1) assert len(lookupd.topics()['topics']) == 0 assert len(lookupd.channels('topic')['channels']) == 0 assert len(lookupd.nodes()['producers']) == 1 nsqd.create_topic('topic') gevent.sleep(0.1) info = lookupd.lookup('topic') assert len(info['channels']) == 0 assert len(info['producers']) == 1 assert len(lookupd.topics()['topics']) == 1 assert len(lookupd.channels('topic')['channels']) == 0 nsqd.create_channel('topic', 'channel') gevent.sleep(0.1) info = lookupd.lookup('topic') assert len(info['channels']) == 1 assert len(info['producers']) == 1 assert len(lookupd.topics()['topics']) == 1 assert len(lookupd.channels('topic')['channels']) == 1
def test_max_concurrency(): server1 = NsqdIntegrationServer() server2 = NsqdIntegrationServer() with server1, server2: class Accounting(object): count = 0 total = 100 concurrency = 0 error = None for server in (server1, server2): conn = Nsqd( address=server.address, tcp_port=server.tcp_port, http_port=server.http_port, ) for _ in range(Accounting.total // 2): conn.publish_http('test', b'danger zone!') reader = Reader( topic='test', channel='test', nsqd_tcp_addresses=[ server1.tcp_address, server2.tcp_address, ], max_in_flight=5, max_concurrency=1, ) @reader.on_exception.connect def error_handler(reader, message, error): if isinstance(error, NSQSocketError): return Accounting.error = error reader.close() @reader.on_message.connect def handler(reader, message): assert message.body == b'danger zone!' assert Accounting.concurrency == 0 Accounting.concurrency += 1 gevent.sleep() Accounting.concurrency -= 1 Accounting.count += 1 if Accounting.count == Accounting.total: reader.close() reader.start() if Accounting.error: raise Accounting.error assert Accounting.count == Accounting.total
def test_lookupd(): with LookupdIntegrationServer() as lookupd_server: server1 = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address) server2 = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address) with server1, server2: class Accounting(object): count = 0 total = 500 concurrency = 0 error = None for server in (server1, server2): conn = Nsqd( address=server.address, tcp_port=server.tcp_port, http_port=server.http_port, ) for _ in range(Accounting.total // 2): conn.publish_http('test', b'danger zone!') reader = Reader( topic='test', channel='test', lookupd_http_addresses=lookupd_server.http_address, max_in_flight=32, ) @reader.on_exception.connect def error_handler(reader, message, error): if isinstance(error, NSQSocketError): return Accounting.error = error reader.close() @reader.on_message.connect def handler(reader, message): assert message.body == b'danger zone!' Accounting.count += 1 if Accounting.count == Accounting.total: reader.close() gevent.sleep(0.1) reader.start() if Accounting.error: raise Accounting.error assert Accounting.count == Accounting.total
def test_topics_channels(): with NsqdIntegrationServer() as server: conn = gnsq.NsqdHTTPClient(server.address, server.http_port) assert len(conn.stats()['topics']) == 0 with pytest.raises(gnsq.errors.NSQHttpError): conn.delete_topic('topic') conn.create_topic('topic') topics = conn.stats()['topics'] assert len(topics) == 1 assert topics[0]['topic_name'] == 'topic' conn.delete_topic('topic') assert len(conn.stats()['topics']) == 0 with pytest.raises(gnsq.errors.NSQHttpError): conn.create_channel('topic', 'channel') with pytest.raises(gnsq.errors.NSQHttpError): conn.delete_channel('topic', 'channel') conn.create_topic('topic') assert len(conn.stats()['topics'][0]['channels']) == 0 conn.create_channel('topic', 'channel') channels = conn.stats()['topics'][0]['channels'] assert len(channels) == 1 assert channels[0]['channel_name'] == 'channel' conn.delete_channel('topic', 'channel') assert len(conn.stats()['topics'][0]['channels']) == 0
def test_backoff(): with NsqdIntegrationServer() as server: conn = Nsqd( address=server.address, tcp_port=server.tcp_port, http_port=server.http_port, ) for _ in range(500): conn.publish_http('test', 'danger zone!') reader = Reader(topic='test', channel='test', nsqd_tcp_addresses=[server.tcp_address], max_in_flight=100, message_handler=lambda reader, message: None) reader.start(block=False) reader.start_backoff() assert reader.state == states.THROTTLED assert reader.total_in_flight_or_ready <= 1 reader.complete_backoff() assert reader.state == states.RUNNING
def test_lookup(): lookupd_server = LookupdIntegrationServer() nsqd_server = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address) @with_all(lookupd_server, nsqd_server) def _(lookupd_server, nsqd_server): lookupd = gnsq.Lookupd(lookupd_server.http_address) conn = gnsq.Nsqd(nsqd_server.address, http_port=nsqd_server.http_port) assert len(lookupd.topics()['topics']) == 0 assert len(lookupd.channels('topic')['channels']) == 0 assert len(lookupd.nodes()['producers']) == 1 conn.create_topic('topic') gevent.sleep(0.1) info = lookupd.lookup('topic') assert len(info['channels']) == 0 assert len(info['producers']) == 1 assert len(lookupd.topics()['topics']) == 1 assert len(lookupd.channels('topic')['channels']) == 0 conn.create_channel('topic', 'channel') gevent.sleep(0.1) info = lookupd.lookup('topic') assert len(info['channels']) == 1 assert len(info['producers']) == 1 assert len(lookupd.topics()['topics']) == 1 assert len(lookupd.channels('topic')['channels']) == 1
def test_publish(): with NsqdIntegrationServer() as server: conn = gnsq.NsqdHTTPClient(server.address, server.http_port) conn.publish('topic', b'sup') assert conn.stats()['topics'][0]['depth'] == 1 conn.multipublish('topic', [b'sup', b'sup']) assert conn.stats()['topics'][0]['depth'] == 3 conn.multipublish('topic', iter([b'sup', b'sup', b'sup'])) assert conn.stats()['topics'][0]['depth'] == 6 conn.empty_topic('topic') assert conn.stats()['topics'][0]['depth'] == 0 conn.create_topic('topic') conn.create_channel('topic', 'channel') conn.publish('topic', b'sup') assert conn.stats()['topics'][0]['channels'][0]['depth'] == 1 conn.empty_channel('topic', 'channel') assert conn.stats()['topics'][0]['channels'][0]['depth'] == 0 if server.version < (0, 3, 6): return conn.publish('topic', b'sup', 60 * 1000) stats = conn.stats() assert stats['topics'][0]['channels'][0]['depth'] == 0 assert stats['topics'][0]['channels'][0]['deferred_count'] == 1
def test_tls_publish(): extra_params = [ '--tls-required', 'true', '--https-address', '127.0.0.1:4152', ] with NsqdIntegrationServer(extra_params=extra_params) as server: producer = Producer( server.tcp_address, tls_options={ 'keyfile': server.tls_key, 'certfile': server.tls_cert, }, tls_v1=True, ) producer.start() for _ in range(100): producer.publish('test', b'hi') producer.close() producer.join() conn = NsqdHTTPClient( server.address, '4152', connection_class=urllib3.HTTPSConnectionPool, cert_reqs='CERT_NONE', ) stats = conn.stats() assert stats['topics'][0]['depth'] == 100
def test_publish_error(): with NsqdIntegrationServer() as server: producer = Producer(server.tcp_address) producer.start() with pytest.raises(NSQInvalid): producer.publish('test', b'hi', defer=-1000) producer.close() producer.join()
def test_cls_error(): with NsqdIntegrationServer() as server: conn = Nsqd(address=server.address, tcp_port=server.tcp_port) conn.connect() assert conn.state == states.CONNECTED conn.close() frame, error = conn.read_response() assert frame == nsq.FRAME_TYPE_ERROR assert isinstance(error, errors.NSQInvalid)
def test_backoff(): with NsqdIntegrationServer() as server: conn = NsqdHTTPClient(server.address, server.http_port) for _ in range(500): conn.publish('test', 'danger zone!') consumer = Consumer(topic='test', channel='test', nsqd_tcp_addresses=[server.tcp_address], max_in_flight=100, message_handler=lambda consumer, message: None) consumer.start(block=False) consumer._redistributed_ready_event.wait() conn = next(iter(consumer._connections)) consumer._message_backoffs[conn].failure() consumer._message_backoffs[conn].failure() consumer._start_backoff(conn) consumer._redistribute_ready_state() assert consumer._connections[conn] == states.BACKOFF assert consumer.total_ready_count == 0 consumer._start_throttled(conn) consumer._redistribute_ready_state() consumer._redistribute_ready_state() assert consumer._connections[conn] == states.THROTTLED assert consumer.total_ready_count == 1 consumer._message_backoffs[conn].success() consumer._complete_backoff(conn) consumer._redistribute_ready_state() assert consumer._connections[conn] == states.BACKOFF assert consumer.total_ready_count == 0 consumer._start_throttled(conn) consumer._redistribute_ready_state() assert consumer._connections[conn] == states.THROTTLED assert consumer.total_ready_count == 1 consumer._message_backoffs[conn].success() consumer._complete_backoff(conn) consumer._redistribute_ready_state() assert consumer._connections[conn] == states.RUNNING assert consumer.total_ready_count == 100
def test_multipublish(): with NsqdIntegrationServer() as server: producer = Producer(server.tcp_address) producer.start() for _ in range(10): producer.multipublish('test', 10 * [b'hi']) producer.close() producer.join() conn = NsqdHTTPClient(server.address, server.http_port) stats = conn.stats() assert stats['topics'][0]['depth'] == 100
def test_messages(): with NsqdIntegrationServer() as server: class Accounting(object): count = 0 total = 500 error = None conn = Nsqd( address=server.address, tcp_port=server.tcp_port, http_port=server.http_port, ) for _ in range(Accounting.total): conn.publish_http('test', b'danger zone!') reader = Reader( topic='test', channel='test', nsqd_tcp_addresses=[server.tcp_address], max_in_flight=100, ) @reader.on_exception.connect def error_handler(reader, message, error): if isinstance(error, NSQSocketError): return Accounting.error = error reader.close() @reader.on_message.connect def handler(reader, message): assert message.body == b'danger zone!' Accounting.count += 1 if Accounting.count == Accounting.total: assert not reader.is_starved reader.close() reader.start() if Accounting.error: raise Accounting.error assert Accounting.count == Accounting.total
def test_async_publish(): with NsqdIntegrationServer() as server: results = [] producer = Producer(server.tcp_address) producer.start() for _ in range(100): results.append(producer.publish('test', b'hi', raise_error=False)) gevent.joinall(results, raise_error=True) producer.close() producer.join() conn = NsqdHTTPClient(server.address, server.http_port) stats = conn.stats() assert stats['topics'][0]['depth'] == 100
def test_publish(): with NsqdIntegrationServer() as server: conn = gnsq.Nsqd(server.address, http_port=server.http_port) conn.publish('topic', b'sup') assert conn.stats()['topics'][0]['depth'] == 1 conn.multipublish('topic', ['sup', 'sup']) assert conn.stats()['topics'][0]['depth'] == 3 conn.multipublish('topic', iter(['sup', 'sup', 'sup'])) assert conn.stats()['topics'][0]['depth'] == 6 conn.empty_topic('topic') assert conn.stats()['topics'][0]['depth'] == 0 conn.create_topic('topic') conn.create_channel('topic', 'channel') conn.publish('topic', b'sup') assert conn.stats()['topics'][0]['channels'][0]['depth'] == 1 conn.empty_channel('topic', 'channel') assert conn.stats()['topics'][0]['channels'][0]['depth'] == 0
def test_socket_upgrades(tls, deflate, snappy): with NsqdIntegrationServer() as server: options = { 'address': server.address, 'tcp_port': server.tcp_port, 'deflate': deflate, 'snappy': snappy, } if tls: options.update({ 'tls_v1': True, 'tls_options': { 'keyfile': server.tls_key, 'certfile': server.tls_cert, } }) conn = Nsqd(**options) conn.connect() assert conn.state == states.CONNECTED if deflate and snappy: with pytest.raises(errors.NSQErrorCode): conn.identify() return if tls and BAD_GEVENT: with pytest.raises(AttributeError): conn.identify() return if tls and server.version < (0, 2, 28): with pytest.raises(ssl.SSLError): conn.identify() return resp = conn.identify() assert isinstance(resp, dict) assert resp['tls_v1'] is tls assert resp['deflate'] is deflate assert resp['snappy'] is snappy if tls and (deflate or snappy): assert isinstance(conn.stream.socket._socket, SSLSocket) elif tls: assert isinstance(conn.stream.socket, SSLSocket) if deflate: assert isinstance(conn.stream.socket, DefalteSocket) if snappy: assert isinstance(conn.stream.socket, SnappySocket) conn.publish('topic', b'sup') frame, data = conn.read_response() assert frame == nsq.FRAME_TYPE_RESPONSE assert data == b'OK' conn.subscribe('topic', 'channel') frame, data = conn.read_response() assert frame == nsq.FRAME_TYPE_RESPONSE assert data == b'OK' conn.ready(1) frame, data = conn.read_response() assert frame == nsq.FRAME_TYPE_MESSAGE assert data.body == b'sup' conn.close_stream()
def test_basic(): with NsqdIntegrationServer() as server: conn = gnsq.NsqdHTTPClient(server.address, server.http_port) assert conn.ping() == b'OK' assert 'topics' in conn.stats() assert 'version' in conn.info()