def create_publisher(message_num, thread_num, hostname, port):
global g_publish_finish_time
published = list()
def _on_publish(mosq, obj, mid):
published.append(mid)
#_logger.debug('got publish ack %d' % mid)
def _on_disconnect(mosq, obj, rc):
global g_lost_connection_count
if rc != 0:
_logger.error(
'publisher lost connection to server, will reconnect')
g_lost_connection_count += 1
create_publisher(message_num, thread_num, hostname, port)
pub_id = "publisher-%s" % (socket.gethostname())
_logger.debug("publisher id is (%s)" % pub_id)
client = GossipClient(client_id=pub_id)
client.on_publish = _on_publish
client.on_disconnect = _on_disconnect
client.connect(hostname, port)
for i in xrange(message_num):
for j in xrange(thread_num):
now = time.time()
client.publish(topic="%s-%d" % (socket.gethostname(), j + 1),
payload=str(now),
qos=1)
count = i * thread_num + j
if count % 100 == 0:
_logger.debug("published %d messages" % count)
ret = client.loop(10.0) # Mosquitto won't work without this sleep
if ret != 0:
_logger.fatal("publish loop returned %d, will retry" % ret)
return create_publisher(message_num, thread_num, hostname,
port)
while len(published) < message_num * thread_num:
_logger.debug("published %d" % len(published))
ret = client.loop()
if ret != 0:
_logger.fatal("publisher's connection is lost, rc=%d" % rc)
sys.exit(0)
g_publish_finish_time = time.time()
_logger.info("all messages published")
def create_publisher(message_num, thread_num, hostname, port):
global g_publish_finish_time
published = list()
def _on_publish(mosq, obj, mid):
published.append(mid)
#_logger.debug('got publish ack %d' % mid)
def _on_disconnect(mosq, obj, rc):
global g_lost_connection_count
if rc != 0:
_logger.error('publisher lost connection to server, will reconnect')
g_lost_connection_count += 1
create_publisher(message_num, thread_num, hostname, port)
pub_id = "publisher-%s" % (socket.gethostname())
_logger.debug("publisher id is (%s)" % pub_id)
client = GossipClient(client_id=pub_id)
client.on_publish = _on_publish
client.on_disconnect = _on_disconnect
client.connect(hostname, port)
for i in xrange(message_num):
for j in xrange(thread_num):
now = time.time()
client.publish(topic="%s-%d" % (socket.gethostname(), j+1), payload=str(now), qos=1)
count = i * thread_num + j
if count % 100 == 0:
_logger.debug("published %d messages" % count)
ret = client.loop(10.0) # Mosquitto won't work without this sleep
if ret != 0:
_logger.fatal("publish loop returned %d, will retry" % ret)
return create_publisher(message_num, thread_num, hostname, port)
while len(published) < message_num * thread_num:
_logger.debug("published %d" % len(published))
ret = client.loop()
if ret != 0:
_logger.fatal("publisher's connection is lost, rc=%d" % rc)
sys.exit(0)
g_publish_finish_time = time.time()
_logger.info("all messages published")
class TestWorker(object):
def __init__(self, worker_id, thread_num, hostname, port):
self.client_id = "%s_test_%d" % (socket.gethostname(), worker_id)
self.worker_id = worker_id
self.thread_num = thread_num
self.hostname = hostname
self.port = port
self.client = GossipClient(client_id = self.client_id, clean_session = True)
self.client.on_connect = self.on_connect
self.client.on_disconnect = self.on_disconnect
self.client.on_publish = self.on_publish
self.client.on_subscribe = self.on_subscribe
self.client.on_message = self.on_message
self.client.on_unsubscribe = self.on_unsubscribe
self.subscribe_done = False
def on_connect(self, mosq, obj, rc):
if rc == 0:
_logger.debug('worker %d connected' % self.worker_id)
else:
_logger.error('worker %d on_connect, rc=%d' % (self.worker_id, rc))
def on_disconnect(self, mosq, obj, rc):
global g_lost_connection_count
if rc == 0:
_logger.debug('worker %d disconnected normally' % self.worker_id)
else:
g_lost_connection_count += 1
_logger.error('worker %d lost connection to server' % self.worker_id)
if not self.subscribe_done:
_logger.error('worker %d not done subscription and lost connection, will retry',
self.worker_id)
self._do_stuff(self.message_num)
def on_publish(self, mosq, obj, mid):
return
_logger.debug('on worker %d, published mid=%d' % (self.worker_id, mid))
def on_subscribe(self, mosq, obj, mid, qos_list):
g_subscribe_done.add(self.worker_id)
self.subscribe_done = True
#_logger.debug('on worker %d, subscribed mid=%d' % (self.worker_id, mid))
def on_unsubscribe(self, mosq, obj, mid):
return
_logger.debug('on woker %d, unsubscribed mid=%d' % (self.worker_id, mid))
def on_message(self, mosq, obj, msg):
now = time.time()
sent = float(msg.payload)
elapsed = now - sent
self.status.append(elapsed)
return
_logger.debug('worker %d got message, topic=(%s), payload=(%s)' %
(self.worker_id, msg.topic, msg.payload))
def _do_stuff(self, message_num):
ret = self.client.connect(self.hostname, self.port, keepalive=6000)
if ret != 0:
_logger.error("worker %d connect() returned %d" % (self.worker_id, ret))
return self._do_stuff(message_num)
topic = "%s-%d" % (socket.gethostname(), self.worker_id)
self.subscribe_done = False
self.client.subscribe(topic, qos=1)
count = 0
while len(self.status) < message_num:
count += 1
if count % 100 == 0:
_logger.debug("worker %d get %d messages" % (self.worker_id, len(self.status)))
ret = self.client.loop()
if ret != 0:
_logger.error("worker %d loop() failed, returned %d" % (self.worker_id, ret))
return self._do_stuff(message_num)
now = time.time()
if g_publish_finish_time is not None:
late = (now - g_publish_finish_time)
if late > 60.0:
_logger.debug("worker %d: too long waiting for message, quit with %d received" % (self.worker_id, len(self.status)))
return self._do_stuff(message_num)
else:
_logger.debug("worker %d, %.3f seconds late" % (self.worker_id, late))
g_subscribe_done.add(self.worker_id)
self.subscribe_done = True
self.client.disconnect()
def start_listening(self, message_num):
self.status = self._build_status()
self.message_num = message_num
self._do_stuff(message_num)
def _build_status(self):
return list()
def collect(self):
return self.status
class TestWorker(object):
def __init__(self, worker_id, thread_num, hostname, port):
self.client_id = "%s_test_%d" % (socket.gethostname(), worker_id)
self.worker_id = worker_id
self.thread_num = thread_num
self.hostname = hostname
self.port = port
self.client = GossipClient(client_id=self.client_id,
clean_session=True)
self.client.on_connect = self.on_connect
self.client.on_disconnect = self.on_disconnect
self.client.on_publish = self.on_publish
self.client.on_subscribe = self.on_subscribe
self.client.on_message = self.on_message
self.client.on_unsubscribe = self.on_unsubscribe
self.subscribe_done = False
def on_connect(self, mosq, obj, rc):
if rc == 0:
_logger.debug('worker %d connected' % self.worker_id)
else:
_logger.error('worker %d on_connect, rc=%d' % (self.worker_id, rc))
def on_disconnect(self, mosq, obj, rc):
global g_lost_connection_count
if rc == 0:
_logger.debug('worker %d disconnected normally' % self.worker_id)
else:
g_lost_connection_count += 1
_logger.error('worker %d lost connection to server' %
self.worker_id)
if not self.subscribe_done:
_logger.error(
'worker %d not done subscription and lost connection, will retry',
self.worker_id)
self._do_stuff(self.message_num)
def on_publish(self, mosq, obj, mid):
return
_logger.debug('on worker %d, published mid=%d' % (self.worker_id, mid))
def on_subscribe(self, mosq, obj, mid, qos_list):
g_subscribe_done.add(self.worker_id)
self.subscribe_done = True
#_logger.debug('on worker %d, subscribed mid=%d' % (self.worker_id, mid))
def on_unsubscribe(self, mosq, obj, mid):
return
_logger.debug('on woker %d, unsubscribed mid=%d' %
(self.worker_id, mid))
def on_message(self, mosq, obj, msg):
now = time.time()
sent = float(msg.payload)
elapsed = now - sent
self.status.append(elapsed)
return
_logger.debug('worker %d got message, topic=(%s), payload=(%s)' %
(self.worker_id, msg.topic, msg.payload))
def _do_stuff(self, message_num):
ret = self.client.connect(self.hostname, self.port, keepalive=6000)
if ret != 0:
_logger.error("worker %d connect() returned %d" %
(self.worker_id, ret))
return self._do_stuff(message_num)
topic = "%s-%d" % (socket.gethostname(), self.worker_id)
self.subscribe_done = False
self.client.subscribe(topic, qos=1)
count = 0
while len(self.status) < message_num:
count += 1
if count % 100 == 0:
_logger.debug("worker %d get %d messages" %
(self.worker_id, len(self.status)))
ret = self.client.loop()
if ret != 0:
_logger.error("worker %d loop() failed, returned %d" %
(self.worker_id, ret))
return self._do_stuff(message_num)
now = time.time()
if g_publish_finish_time is not None:
late = (now - g_publish_finish_time)
if late > 60.0:
_logger.debug(
"worker %d: too long waiting for message, quit with %d received"
% (self.worker_id, len(self.status)))
return self._do_stuff(message_num)
else:
_logger.debug("worker %d, %.3f seconds late" %
(self.worker_id, late))
g_subscribe_done.add(self.worker_id)
self.subscribe_done = True
self.client.disconnect()
def start_listening(self, message_num):
self.status = self._build_status()
self.message_num = message_num
self._do_stuff(message_num)
def _build_status(self):
return list()
def collect(self):
return self.status