class NsqPublisher(
AbstractPublisher, ):
"""
使用nsq作为中间件
"""
# noinspection PyAttributeOutsideInit
def custom_init(self):
self._nsqd_cleint = NsqdHTTPClient(frame_config.NSQD_HTTP_CLIENT_HOST,
frame_config.NSQD_HTTP_CLIENT_PORT)
self._producer = Producer(frame_config.NSQD_TCP_ADDRESSES)
self._producer.start()
def concrete_realization_of_publish(self, msg):
# noinspection PyTypeChecker
self._producer.publish(self._queue_name, msg.encode())
def clear(self):
try:
self._nsqd_cleint.empty_topic(self._queue_name)
except NSQHttpError as e:
self.logger.exception(e) # 不能清除一个不存在的topoc会报错，和其他消息队列中间件不同。
self.logger.warning(f'清除 {self._queue_name} topic中的消息成功')
def get_message_count(self):
return self._nsqd_cleint.stats(
self._queue_name)['topics'][0]['message_count']
def close(self):
self._producer.close()
def test_tls_publish():
extra_params = [
'--tls-required', 'true',
'--https-address', '127.0.0.1:4152',
]
with NsqdIntegrationServer(extra_params=extra_params) as server:
producer = Producer(
server.tcp_address,
tls_options={
'keyfile': server.tls_key,
'certfile': server.tls_cert,
},
tls_v1=True,
)
producer.start()
for _ in range(100):
producer.publish('test', b'hi')
producer.close()
producer.join()
conn = NsqdHTTPClient(
server.address,
'4152',
connection_class=urllib3.HTTPSConnectionPool,
cert_reqs='CERT_NONE',
)
stats = conn.stats()
assert stats['topics'][0]['depth'] == 100
def test_publish_error():
with NsqdIntegrationServer() as server:
producer = Producer(server.tcp_address)
producer.start()
with pytest.raises(NSQInvalid):
producer.publish('test', b'hi', defer=-1000)
producer.close()
producer.join()
def test_publish_error():
with NsqdIntegrationServer() as server:
producer = Producer(server.tcp_address)
producer.start()
with pytest.raises(NSQInvalid):
producer.publish('test', b'hi', defer=-1000)
producer.close()
producer.join()
def test_multipublish():
with NsqdIntegrationServer() as server:
producer = Producer(server.tcp_address)
producer.start()
for _ in range(10):
producer.multipublish('test', 10 * [b'hi'])
producer.close()
producer.join()
conn = NsqdHTTPClient(server.address, server.http_port)
stats = conn.stats()
assert stats['topics'][0]['depth'] == 100
def test_multipublish():
with NsqdIntegrationServer() as server:
producer = Producer(server.tcp_address)
producer.start()
for _ in range(10):
producer.multipublish('test', 10 * [b'hi'])
producer.close()
producer.join()
conn = NsqdHTTPClient(server.address, server.http_port)
stats = conn.stats()
assert stats['topics'][0]['depth'] == 100
def test_async_publish():
with NsqdIntegrationServer() as server:
results = []
producer = Producer(server.tcp_address)
producer.start()
for _ in range(100):
results.append(producer.publish('test', b'hi', raise_error=False))
gevent.joinall(results, raise_error=True)
producer.close()
producer.join()
conn = NsqdHTTPClient(server.address, server.http_port)
stats = conn.stats()
assert stats['topics'][0]['depth'] == 100
def test_async_publish():
with NsqdIntegrationServer() as server:
results = []
producer = Producer(server.tcp_address)
producer.start()
for _ in range(100):
results.append(producer.publish('test', b'hi', raise_error=False))
gevent.joinall(results, raise_error=True)
producer.close()
producer.join()
conn = NsqdHTTPClient(server.address, server.http_port)
stats = conn.stats()
assert stats['topics'][0]['depth'] == 100
class Output(cowrie.core.output.Output):
"""
nsq output
"""
def start(self):
host = CowrieConfig().get('output_nsq', 'host')
try:
port = CowrieConfig().getint('output_nsq', 'port')
except NoOptionError:
port = 4150
try:
auth_secret = CowrieConfig().get('output_nsq',
'auth_secret').encode("utf-8")
except NoOptionError:
auth_secret = None
try:
tls = CowrieConfig().getboolean('output_nsq', 'use_tls')
except NoOptionError:
tls = False
self.producer = Producer(host + ":" + str(port),
auth_secret=auth_secret,
tls_v1=tls,
tls_options={})
self.topic = CowrieConfig().get('output_nsq', 'topic')
self.producer.start()
def stop(self):
self.producer.close()
def write(self, logentry):
for i in list(logentry.keys()):
# Remove twisted 15 legacy keys
if i.startswith('log_'):
del logentry[i]
self.producer.publish(self.topic, json.dumps(logentry).encode("utf-8"))