def run(project_id, topic, amount):
"""
Publishes batch of --amount test messages. --topic have corresponding json template in resources/mockups/templates
:param project_id: Project on which topic is created
:param topic: Name of topic to which publish messages
:param amount: Number of messages to publish
:return:
"""
from time import time
from app.tools.mockups import DictFromTemplate
psp = PubSubPublisher(project_id, topic)
mockup_data = DictFromTemplate(topic).generate()
msg_size = dumps(mockup_data).encode().__len__()
time_start = time()
for i in range(amount):
psp.publish_message(mockup_data)
psp.finish()
time_stop = time()
seconds = time_stop - time_start
log.log_info(
"Published {} messages in {:.2f} seconds. That is {:.2f} m/s & {:.2f} MB/s!"
.format(amount, seconds, amount / seconds,
amount * msg_size / (seconds * 1024 * 1024)))
def receive_and_index(self):
subscription_path = self.client.subscription_path(
self.project_id, "{}-subscription-elastic".format(self.topic_name))
def callback(message):
latency = 1000 * (message._received_timestamp -
message.publish_time.timestamp())
message_id = message.message_id
document = self.enrich(
BaseSubscriber.protobuf_to_json(message.data))
self.elasticsearch_index_managers[self.counter %
self.elastic_managers].queue.put(
(document, message_id))
message.ack()
if self.seconds:
self.latencies.append(latency)
self.counter += 1
self.client.subscribe(subscription_path, callback=callback)
# if seconds specified, run only for given time. if not - run indefinitely
if self.seconds:
sleep(self.seconds)
time_queue_join_start = time()
for manager in self.elasticsearch_index_managers:
manager.queue.join()
time_queue_join_stop = time()
self.seconds = self.seconds + time_queue_join_stop - time_queue_join_start
log.log_info(
"{} - Read {} messages in {:.2f} seconds. That is {:.2f} mps!".
format(self.__class__.__name__, self.counter, self.seconds,
self.counter / self.seconds))
if self.latencies:
avg_latency = float(sum(self.latencies)) / float(
len(self.latencies))
log.log_info("{} - Average latency was {:.2f} ms.".format(
self.__class__.__name__, avg_latency))
else:
while True:
sleep(60)
def run(self, threads_number):
if self.seconds:
log.log_info("{} - Running {} threads for {} seconds".format(
self.__class__.__name__, threads_number, self.seconds))
else:
log.log_info("{} - Running {} threads forever".format(
self.__class__.__name__, threads_number))
for i in range(threads_number):
self.threads.append(
self.thread(self.project, self.topic, self.seconds))
for thread in self.threads:
thread.join()
self.counter = sum([t.counter for t in self.threads])
self.time_taken = max([t.seconds for t in self.threads])
log.log_info(
"{} - Summary: Processed {} objects in {:.2f} seconds. That is {:.2f} ops!"
.format(self.__class__.__name__, self.counter, self.time_taken,
self.counter / self.time_taken))
if self.seconds:
try:
avg_latency = float(
sum([sum(t.latencies) for t in self.threads])) / float(
sum([len(t.latencies) for t in self.threads]))
except ZeroDivisionError:
avg_latency = 0
log.log_info("{} - Summary: Average latency was {:.2f} ms.".format(
self.__class__.__name__, avg_latency))
def run(self):
websocket.enableTrace(True)
self.ws = websocket.WebSocketApp(self.url,
on_message=self.on_message,
on_error=self.on_error,
on_close=self.on_close)
self.ws.counter = 0
self.ws.on_open = self.on_open
log.log_info(
"{} - Starting listening on: {} Messages will be published to: {}".
format(self.__class__.__name__, self.url, str(self.publisher)))
self.ws.run_forever()
def run(*args):
if self.seconds:
log.log_info("{} - Running for {} seconds".format(
self.__class__.__name__, self.seconds))
sleep(self.seconds)
self.ws.close()
log.log_info(
"{} - Published {} messages in {} seconds. That is {:.2f} mps!"
.format(self.__class__.__name__, self.ws.counter,
self.seconds, self.ws.counter / self.seconds))
else:
log.log_info("{} - Running forever".format(
self.__class__.__name__, self.seconds))
while True:
pass
def on_close(self):
log.log_info("{} - Websocket closed".format(self.__class__.__name__))
def finish(self):
log.log_info("{} - Processed results: {}".format(
self.__class__.__name__, self.results_counter))