def _check_memory(self):
if not self.pmi:
self.pmi = ProcessMemoryInfo()
self.last_vsz = self.pmi.vsz
self.initial_vsz = self.pmi.vsz
utc = datetime.datetime.utcnow()
check = self.last_time is None
if self.last_time:
diff = utc - self.last_time
if diff.seconds > 30:
check = True
if check:
self.last_time = utc
self.pmi.update()
diff = (self.pmi.vsz - self.last_vsz) / 1000
idiff = (self.pmi.vsz - self.initial_vsz) / 1000
self.total_processed += self.processed
per_message = 0
if self.total_processed:
per_message = idiff / self.total_processed
LOG.debug("%20s %6dk/%6dk ram, "
"%3d/%4d msgs @ %6dk/msg" %
(self.name, diff, idiff, self.processed,
self.total_processed, per_message))
self.last_vsz = self.pmi.vsz
self.processed = 0
def process_request(self, request: HttpRequest) -> HttpResponse:
self._tracker = ClassTracker()
for cls in apps.get_models() + self.classes:
self._tracker.track_class(cls)
self._tracker.create_snapshot('before')
self.record_stats({'before': ProcessMemoryInfo()})
response = super(MemoryPanel, self).process_request(request)
self.record_stats({'after': ProcessMemoryInfo()})
self._tracker.create_snapshot('after')
stats = self._tracker.stats
stats.annotate()
self.record_stats({'stats': stats})
return response
def _check_memory(self):
if not self.pmi:
self.pmi = ProcessMemoryInfo()
self.last_vsz = self.pmi.vsz
self.initial_vsz = self.pmi.vsz
utc = datetime.datetime.utcnow()
check = self.last_time is None
if self.last_time:
diff = utc - self.last_time
if diff.seconds > 30:
check = True
if check:
self.last_time = utc
self.pmi.update()
diff = (self.pmi.vsz - self.last_vsz) / 1000
idiff = (self.pmi.vsz - self.initial_vsz) / 1000
self.total_processed += self.processed
per_message = 0
if self.total_processed:
per_message = idiff / self.total_processed
_get_child_logger().debug("%20s %20s %6dk/%6dk ram, "
"%3d/%4d msgs @ %6dk/msg" %
(self.name, self.exchange, diff, idiff, self.processed,
self.total_processed, per_message))
self.last_vsz = self.pmi.vsz
self.processed = 0
def profiler_handler(req):
self._tracker.create_snapshot('before')
before = ProcessMemoryInfo()
req_before = get_memory_object(req.registry)
try:
result = handler(req)
except:
raise
finally:
after = ProcessMemoryInfo()
self._tracker.create_snapshot('after')
class_stats = self._tracker.stats
class_stats.annotate()
self.stats = dict(
before=before,
after=after,
class_stats=class_stats,
req_before=req_before,
req_after=get_memory_object(req.registry),
)
return result
def _getProcessMemory():
"""Utility function that defined the logic to get memory."""
return ProcessMemoryInfo().rss
def process_response(self, request, response):
self.record_stats({'after': ProcessMemoryInfo()})
self._tracker.create_snapshot('after')
stats = self._tracker.stats
stats.annotate()
self.record_stats({'stats': stats})
def process_request(self, request):
self._tracker = ClassTracker()
for cls in apps.get_models() + self.classes:
self._tracker.track_class(cls)
self._tracker.create_snapshot('before')
self.record_stats({'before': ProcessMemoryInfo()})
class NovaConsumer(kombu.mixins.ConsumerMixin):
def __init__(self, name, connection, deployment, durable, queue_arguments):
self.connection = connection
self.deployment = deployment
self.durable = durable
self.queue_arguments = queue_arguments
self.name = name
self.last_time = None
self.pmi = None
self.processed = 0
self.total_processed = 0
def _create_exchange(self, name, type, exclusive=False, auto_delete=False):
return kombu.entity.Exchange(name,
type=type,
exclusive=exclusive,
durable=self.durable,
auto_delete=auto_delete)
def _create_queue(self,
name,
nova_exchange,
routing_key,
exclusive=False,
auto_delete=False):
return kombu.Queue(name,
nova_exchange,
durable=self.durable,
auto_delete=exclusive,
exclusive=auto_delete,
queue_arguments=self.queue_arguments,
routing_key=routing_key)
def get_consumers(self, Consumer, channel):
nova_exchange = self._create_exchange("nova", "topic")
nova_queues = [
self._create_queue('monitor.info', nova_exchange, 'monitor.info'),
self._create_queue('monitor.error', nova_exchange, 'monitor.error')
]
return [Consumer(queues=nova_queues, callbacks=[self.on_nova])]
def _process(self, message):
routing_key = message.delivery_info['routing_key']
body = str(message.body)
args = (routing_key, json.loads(body))
asJson = json.dumps(args)
# save raw and ack the message
raw = views.process_raw_data(self.deployment, args, asJson)
if raw:
self.processed += 1
message.ack()
views.post_process(raw, args[1])
self._check_memory()
def _check_memory(self):
if not self.pmi:
self.pmi = ProcessMemoryInfo()
self.last_vsz = self.pmi.vsz
self.initial_vsz = self.pmi.vsz
utc = datetime.datetime.utcnow()
check = self.last_time is None
if self.last_time:
diff = utc - self.last_time
if diff.seconds > 30:
check = True
if check:
self.last_time = utc
self.pmi.update()
diff = (self.pmi.vsz - self.last_vsz) / 1000
idiff = (self.pmi.vsz - self.initial_vsz) / 1000
self.total_processed += self.processed
per_message = 0
if self.total_processed:
per_message = idiff / self.total_processed
LOG.debug("%20s %6dk/%6dk ram, "
"%3d/%4d msgs @ %6dk/msg" %
(self.name, diff, idiff, self.processed,
self.total_processed, per_message))
self.last_vsz = self.pmi.vsz
self.processed = 0
def on_nova(self, body, message):
try:
self._process(message)
except Exception, e:
LOG.debug("Problem: %s\nFailed message body:\n%s" %
(e, json.loads(str(message.body))))
raise
def root():
"""Get overview."""
pmi = ProcessMemoryInfo()
return dict(processinfo=pmi)
def process_request(self, request):
self._tracker = ClassTracker()
for cls in get_models() + self.classes:
self._tracker.track_class(cls, keep=True)
self._tracker.create_snapshot('before')
self._before = ProcessMemoryInfo()
class Consumer(kombu.mixins.ConsumerMixin):
def __init__(self,
name,
connection,
deployment,
durable,
queue_arguments,
exchange,
topics,
connect_max_retries=10,
stats=None):
self.connect_max_retries = connect_max_retries
self.retry_attempts = 0
self.connection = connection
self.deployment = deployment
self.durable = durable
self.queue_arguments = queue_arguments
self.name = name
self.last_time = None
self.pmi = None
self.processed = 0
self.total_processed = 0
self.topics = topics
self.exchange = exchange
if stats is not None:
self.stats = stats
else:
self.stats = dict()
signal.signal(signal.SIGTERM, self._shutdown)
def _create_exchange(self, name, type, exclusive=False, auto_delete=False):
return message_service.create_exchange(name,
exchange_type=type,
exclusive=exclusive,
durable=self.durable,
auto_delete=auto_delete)
def _create_queue(self,
name,
nova_exchange,
routing_key,
exclusive=False,
auto_delete=False):
return message_service.create_queue(
name,
nova_exchange,
durable=self.durable,
auto_delete=exclusive,
exclusive=auto_delete,
queue_arguments=self.queue_arguments,
routing_key=routing_key)
def get_consumers(self, Consumer, channel):
exchange = self._create_exchange(self.exchange, "topic")
queues = [
self._create_queue(topic['queue'], exchange, topic['routing_key'])
for topic in self.topics
]
return [Consumer(queues=queues, callbacks=[self.on_nova])]
def _process(self, message):
routing_key = message.delivery_info['routing_key']
body = str(message.body)
args = (routing_key, json.loads(body))
asJson = json.dumps(args)
# save raw and ack the message
raw, notif = views.process_raw_data(self.deployment, args, asJson,
self.exchange)
self.processed += 1
message.ack()
POST_PROCESS_METHODS[raw.get_name()](raw, notif)
self._check_memory()
def _check_memory(self):
if not self.pmi:
self.pmi = ProcessMemoryInfo()
self.last_vsz = self.pmi.vsz
self.initial_vsz = self.pmi.vsz
utc = datetime.datetime.utcnow()
check = self.last_time is None
if self.last_time:
diff = utc - self.last_time
if diff.seconds > 30:
check = True
if check:
self.last_time = utc
self.pmi.update()
diff = (self.pmi.vsz - self.last_vsz) / 1000
idiff = (self.pmi.vsz - self.initial_vsz) / 1000
self.total_processed += self.processed
per_message = 0
if self.total_processed:
per_message = idiff / self.total_processed
_get_child_logger().debug(
"%20s %20s %6dk/%6dk ram, "
"%3d/%4d msgs @ %6dk/msg" %
(self.name, self.exchange, diff, idiff, self.processed,
self.total_processed, per_message))
self.stats['timestamp'] = utc
self.stats['total_processed'] = self.total_processed
self.stats['processed'] = self.processed
self.last_vsz = self.pmi.vsz
self.processed = 0
def on_nova(self, body, message):
try:
self._process(message)
except ValueError, e:
_get_child_logger().error(
"Error: %s\nMalformed message body found : \n%s" %
(e, str(message.body)))
# Mark message as read to avoid re-reading the malformed message.
message.ack()
except Exception, e:
_get_child_logger().error("Problem: %s\nFailed message body:\n%s" %
(e, str(message.body)))
raise
def track_memory_wrapper(*args, **kwargs):
memory_info = {}
tracker = ClassTracker()
for cls in apps.get_models() + [Context, Template]:
# track all models from registered apps, plus some standard Django ones
tracker.track_class(cls)
try:
tracker.create_snapshot("before")
memory_info["before"] = ProcessMemoryInfo()
result = fn(*args, **kwargs)
memory_info["after"] = ProcessMemoryInfo()
tracker.create_snapshot("after")
memory_info["stats"] = tracker.stats
memory_info["stats"].annotate()
return result
finally:
# record a whole bunch of memory statistics...
resources = [
("resident set size", memory_info["after"].rss),
("virtual size", memory_info["after"].vsz),
]
resources.extend(memory_info["after"] - memory_info["before"])
resources = [(k, pp(v)) for k, v in resources]
resources.extend(memory_info["after"].os_specific)
# record each tracked class as of the final snapshot...
classes_stats = []
snapshot = memory_info["stats"].snapshots[-1]
for class_name in memory_info["stats"].tracked_classes:
# history is a list of tuples that is updated on every creation/deletions: (timestamp, n_instances)
history = [
n for _, n in memory_info["stats"].history[class_name]
]
if history:
classes_stats.append({
"name":
class_name,
"n_instances":
len(history),
"min_instances":
min(history),
"max_instances":
max(history),
"size":
pp(
snapshot.classes.get(class_name,
{}).get("sum", 0)),
})
if not path:
stream = sys.stdout
else:
stream = open(path, "w")
print("\nRESOURCES", file=stream)
for k, v in resources:
print(f"{k:<26}: {v:>10}", file=stream)
print("\nCLASSES", file=stream)
for class_stats in classes_stats:
print(
"{name}: created/deleted {n_instances} times for a min/max of {min_instances}/{max_instances} instances: {size:>10}"
.format(**class_stats),
file=stream,
)
stream.closed
tracker.detach_all_classes()
class NovaConsumer(kombu.mixins.ConsumerMixin):
def __init__(self, name, connection, deployment, durable, queue_arguments):
self.connection = connection
self.deployment = deployment
self.durable = durable
self.queue_arguments = queue_arguments
self.name = name
self.last_time = None
self.pmi = None
self.processed = 0
self.total_processed = 0
def _create_exchange(self, name, type, exclusive=False, auto_delete=False):
return kombu.entity.Exchange(name, type=type, exclusive=exclusive,
durable=self.durable, auto_delete=auto_delete)
def _create_queue(self, name, nova_exchange, routing_key, exclusive=False,
auto_delete=False):
return kombu.Queue(name, nova_exchange, durable=self.durable,
auto_delete=exclusive, exclusive=auto_delete,
queue_arguments=self.queue_arguments,
routing_key=routing_key)
def get_consumers(self, Consumer, channel):
nova_exchange = self._create_exchange("nova", "topic")
nova_queues = [
self._create_queue('monitor.info', nova_exchange, 'monitor.info'),
self._create_queue('monitor.error', nova_exchange, 'monitor.error')
]
return [Consumer(queues=nova_queues, callbacks=[self.on_nova])]
def _process(self, message):
routing_key = message.delivery_info['routing_key']
body = str(message.body)
args = (routing_key, json.loads(body))
asJson = json.dumps(args)
raw = views.process_raw_data(self.deployment, args, asJson)
if raw:
self.processed += 1
self._check_memory()
def _check_memory(self):
if not self.pmi:
self.pmi = ProcessMemoryInfo()
self.last_vsz = self.pmi.vsz
self.initial_vsz = self.pmi.vsz
utc = datetime.datetime.utcnow()
check = self.last_time is None
if self.last_time:
diff = utc - self.last_time
if diff.seconds > 30:
check = True
if check:
self.last_time = utc
self.pmi.update()
diff = (self.pmi.vsz - self.last_vsz) / 1000
idiff = (self.pmi.vsz - self.initial_vsz) / 1000
self.total_processed += self.processed
per_message = 0
if self.total_processed:
per_message = idiff / self.total_processed
LOG.debug("%20s %6dk/%6dk ram, "
"%3d/%4d msgs @ %6dk/msg" %
(self.name, diff, idiff, self.processed,
self.total_processed, per_message))
self.last_vsz = self.pmi.vsz
self.processed = 0
def on_nova(self, body, message):
try:
self._process(message)
except Exception, e:
LOG.exception("Problem %s" % e)
message.ack()
def process_response(self, request, response):
self._after = ProcessMemoryInfo()
self._tracker.create_snapshot('after')
def process():
"""Get process overview."""
pmi = ProcessMemoryInfo()
threads = get_current_threads()
return dict(info=pmi, threads=threads)
class Consumer(kombu.mixins.ConsumerMixin):
def __init__(self, name, connection, deployment, durable, queue_arguments,
exchange, topics, connect_max_retries=10):
self.connect_max_retries = connect_max_retries
self.retry_attempts = 0
self.connection = connection
self.deployment = deployment
self.durable = durable
self.queue_arguments = queue_arguments
self.name = name
self.last_time = None
self.pmi = None
self.processed = 0
self.total_processed = 0
self.topics = topics
self.exchange = exchange
signal.signal(signal.SIGTERM, self._shutdown)
def _create_exchange(self, name, type, exclusive=False, auto_delete=False):
return message_service.create_exchange(name, exchange_type=type,
exclusive=exclusive,
durable=self.durable,
auto_delete=auto_delete)
def _create_queue(self, name, nova_exchange, routing_key, exclusive=False,
auto_delete=False):
return message_service.create_queue(
name, nova_exchange, durable=self.durable, auto_delete=exclusive,
exclusive=auto_delete, queue_arguments=self.queue_arguments,
routing_key=routing_key)
def get_consumers(self, Consumer, channel):
exchange = self._create_exchange(self.exchange, "topic")
queues = [self._create_queue(topic['queue'], exchange,
topic['routing_key'])
for topic in self.topics]
return [Consumer(queues=queues, callbacks=[self.on_nova])]
def _process(self, message):
routing_key = message.delivery_info['routing_key']
body = str(message.body)
args = (routing_key, json.loads(body))
asJson = json.dumps(args)
# save raw and ack the message
raw, notif = views.process_raw_data(
self.deployment, args, asJson, self.exchange)
self.processed += 1
message.ack()
POST_PROCESS_METHODS[raw.get_name()](raw, notif)
self._check_memory()
def _check_memory(self):
if not self.pmi:
self.pmi = ProcessMemoryInfo()
self.last_vsz = self.pmi.vsz
self.initial_vsz = self.pmi.vsz
utc = datetime.datetime.utcnow()
check = self.last_time is None
if self.last_time:
diff = utc - self.last_time
if diff.seconds > 30:
check = True
if check:
self.last_time = utc
self.pmi.update()
diff = (self.pmi.vsz - self.last_vsz) / 1000
idiff = (self.pmi.vsz - self.initial_vsz) / 1000
self.total_processed += self.processed
per_message = 0
if self.total_processed:
per_message = idiff / self.total_processed
_get_child_logger().debug("%20s %20s %6dk/%6dk ram, "
"%3d/%4d msgs @ %6dk/msg" %
(self.name, self.exchange, diff, idiff, self.processed,
self.total_processed, per_message))
self.last_vsz = self.pmi.vsz
self.processed = 0
def on_nova(self, body, message):
try:
self._process(message)
except Exception, e:
_get_child_logger().debug("Problem: %s\nFailed message body:\n%s" %
(e, json.loads(str(message.body))))
raise
class NovaConsumer(object):
def __init__(self, name, connection, deployment, durable):
self.connection = connection
self.deployment = deployment
self.durable = durable
self.name = name
self.last_time = None
self.pmi = None
self.processed = 0
self.total_processed = 0
self.channel = connection.channel()
self.nova_exchange = kombu.entity.Exchange("nova", type="topic",
exclusive=False, durable=self.durable,
auto_delete=False)
self.nova_queues = [
kombu.Queue("stacktash.notifications.info", self.nova_exchange,
durable=self.durable, auto_delete=False,
exclusive=False, routing_key='notifications.info'),
kombu.Queue("stacktash.notifications.error", self.nova_exchange,
durable=self.durable, auto_delete=False,
exclusive=False, routing_key='notifications.error'),
]
def run(self):
self.consumer = Consumer(channel=self.channel,
queues=self.nova_queues,
callbacks=[self.on_nova])
while True:
self.consumer.consume()
self.connection.drain_events()
eventlet.sleep(0)
def _process(self, body, message):
routing_key = message.delivery_info['routing_key']
payload = (routing_key, body)
# make sure jsonable body.
json.dumps(payload)
body = str(message.body)
args = (routing_key, json.loads(body))
asJson = json.dumps(args)
raw = dbapi.process_raw_data(self.deployment, args, asJson)
if raw:
self.processed += 1
self._check_memory()
def _check_memory(self):
if not self.pmi:
self.pmi = ProcessMemoryInfo()
self.last_vsz = self.pmi.vsz
self.initial_vsz = self.pmi.vsz
utc = datetime.datetime.utcnow()
check = self.last_time is None
if self.last_time:
diff = utc - self.last_time
if diff.seconds > 30:
check = True
if check:
self.last_time = utc
self.pmi.update()
diff = (self.pmi.vsz - self.last_vsz) / 1000
idiff = (self.pmi.vsz - self.initial_vsz) / 1000
self.total_processed += self.processed
per_message = 0
if self.total_processed:
per_message = idiff / self.total_processed
LOG.debug("%20s %6dk/%6dk ram, "
"%3d/%4d msgs @ %6dk/msg" %
(self.name, diff, idiff, self.processed,
self.total_processed, per_message))
self.last_vsz = self.pmi.vsz
self.processed = 0
def on_nova(self, body, message):
try:
self._process(body, message)
except Exception, e:
LOG.exception("Problem %s" % e)
finally: