def get_ids():
# prints id (PID) of running clusters
stat = Stat.get_all()
if stat:
for s in stat:
print(s.cluster_id)
else:
print('No clusters appear to be running.')
return True
def test_bad_secret(broker, monkeypatch):
broker.list_key = 'test_bad_secret:q'
async('math.copysign', 1, -1, broker=broker)
stop_event = Event()
stop_event.set()
start_event = Event()
s = Sentinel(stop_event, start_event, broker=broker, start=False)
Stat(s).save()
# change the SECRET
monkeypatch.setattr(Conf, "SECRET_KEY", "OOPS")
stat = Stat.get_all()
assert len(stat) == 0
assert Stat.get(s.parent_pid) is None
task_queue = Queue()
pusher(task_queue, stop_event, broker=broker)
result_queue = Queue()
task_queue.put('STOP')
worker(task_queue, result_queue, Value('f', -1), )
assert result_queue.qsize() == 0
broker.delete_queue()
def stop(self):
Stat(self).save()
name = current_process().name
logger.info(_(f"{name} stopping cluster processes"))
# Stopping pusher
self.event_out.set()
# Wait for it to stop
while self.pusher.is_alive():
sleep(0.1)
Stat(self).save()
# Put poison pills in the queue
for __ in range(len(self.pool)):
self.task_queue.put("STOP")
self.task_queue.close()
# wait for the task queue to empty
self.task_queue.join_thread()
# Wait for all the workers to exit
while len(self.pool):
for p in self.pool:
if not p.is_alive():
self.pool.remove(p)
sleep(0.1)
Stat(self).save()
# Finally stop the monitor
self.result_queue.put("STOP")
self.result_queue.close()
# Wait for the result queue to empty
self.result_queue.join_thread()
logger.info(_(f"{name} waiting for the monitor."))
# Wait for everything to close or time out
count = 0
if not self.timeout:
self.timeout = 30
while self.status() == Conf.STOPPING and count < self.timeout * 10:
sleep(0.1)
Stat(self).save()
count += 1
# Final status
Stat(self).save()
def spawn_cluster(self):
self.pool = []
Stat(self).save()
close_old_django_connections()
# spawn worker pool
for __ in range(self.pool_size):
self.spawn_worker()
# spawn auxiliary
self.monitor = self.spawn_monitor()
self.pusher = self.spawn_pusher()
# set worker cpu affinity if needed
if psutil and Conf.CPU_AFFINITY:
set_cpu_affinity(Conf.CPU_AFFINITY, [w.pid for w in self.pool])
def guard(self):
logger.info(
_(
f"{current_process().name} guarding cluster {humanize(self.cluster_id.hex)}"
)
)
self.start_event.set()
Stat(self).save()
logger.info(_(f"Q Cluster {humanize(self.cluster_id.hex)} running."))
counter = 0
cycle = Conf.GUARD_CYCLE # guard loop sleep in seconds
# Guard loop. Runs at least once
while not self.stop_event.is_set() or not counter:
# Check Workers
for p in self.pool:
with p.timer.get_lock():
# Are you alive?
if not p.is_alive() or p.timer.value == 0:
self.reincarnate(p)
continue
# Decrement timer if work is being done
if p.timer.value > 0:
p.timer.value -= cycle
# Check Monitor
if not self.monitor.is_alive():
self.reincarnate(self.monitor)
# Check Pusher
if not self.pusher.is_alive():
self.reincarnate(self.pusher)
# Call scheduler once a minute (or so)
counter += cycle
if counter >= 30 and Conf.SCHEDULER:
counter = 0
scheduler(broker=self.broker)
# Save current status
Stat(self).save()
sleep(cycle)
self.stop()
def guard(self):
logger.info(
_('{} guarding cluster at {}').format(current_process().name,
self.pid))
self.start_event.set()
Stat(self).save()
logger.info(_('Q Cluster-{} running.').format(self.parent_pid))
scheduler(broker=self.broker)
counter = 0
cycle = 0.5 # guard loop sleep in seconds
# Guard loop. Runs at least once
while not self.stop_event.is_set() or not counter:
# Check Workers
for p in self.pool:
# Are you alive?
if not p.is_alive() or (self.timeout and p.timer.value == 0):
self.reincarnate(p)
continue
# Decrement timer if work is being done
if self.timeout and p.timer.value > 0:
p.timer.value -= cycle
# Check Monitor
if not self.monitor.is_alive():
self.reincarnate(self.monitor)
# Check Pusher
if not self.pusher.is_alive():
self.reincarnate(self.pusher)
# Call scheduler once a minute (or so)
counter += cycle
if counter == 30 and Conf.SCHEDULER:
counter = 0
scheduler(broker=self.broker)
# Save current status
Stat(self).save()
sleep(cycle)
self.stop()
def test_bad_secret(broker, monkeypatch):
broker.list_key = 'test_bad_secret:q'
async ('math.copysign', 1, -1, broker=broker)
stop_event = Event()
stop_event.set()
start_event = Event()
s = Sentinel(stop_event, start_event, broker=broker, start=False)
Stat(s).save()
# change the SECRET
monkeypatch.setattr(Conf, "SECRET_KEY", "OOPS")
stat = Stat.get_all()
assert len(stat) == 0
assert Stat.get(s.parent_pid) is None
task_queue = Queue()
pusher(task_queue, stop_event, broker=broker)
result_queue = Queue()
task_queue.put('STOP')
worker(
task_queue,
result_queue,
Value('f', -1),
)
assert result_queue.qsize() == 0
broker.delete_queue()
def test_monitor(monkeypatch):
assert Stat.get(0).sentinel == 0
c = Cluster()
c.start()
stats = monitor(run_once=True)
c.stop()
assert len(stats) > 0
found_c = False
for stat in stats:
if stat.cluster_id == c.pid:
found_c = True
assert stat.uptime() > 0
assert stat.empty_queues() is True
break
assert found_c is True
# test lock size
monkeypatch.setattr(Conf, 'ORM', 'default')
b = get_broker('monitor_test')
b.enqueue('test')
b.dequeue()
assert b.lock_size() == 1
monitor(run_once=True, broker=b)
b.delete_queue()
def monitor(run_once=False, broker=None):
if not broker:
broker = get_broker()
term = Terminal()
broker.ping()
with term.fullscreen(), term.hidden_cursor(), term.cbreak():
val = None
start_width = int(term.width / 8)
while val not in (
"q",
"Q",
):
col_width = int(term.width / 8)
# In case of resize
if col_width != start_width:
print(term.clear())
start_width = col_width
print(
term.move(0, 0) +
term.black_on_green(term.center(_("Host"), width=col_width -
1)))
print(
term.move(0, 1 * col_width) +
term.black_on_green(term.center(_("Id"), width=col_width - 1)))
print(
term.move(0, 2 * col_width) + term.black_on_green(
term.center(_("State"), width=col_width - 1)))
print(
term.move(0, 3 * col_width) +
term.black_on_green(term.center(_("Pool"), width=col_width -
1)))
print(
term.move(0, 4 * col_width) +
term.black_on_green(term.center(_("TQ"), width=col_width - 1)))
print(
term.move(0, 5 * col_width) +
term.black_on_green(term.center(_("RQ"), width=col_width - 1)))
print(
term.move(0, 6 * col_width) +
term.black_on_green(term.center(_("RC"), width=col_width - 1)))
print(
term.move(0, 7 * col_width) +
term.black_on_green(term.center(_("Up"), width=col_width - 1)))
i = 2
stats = Stat.get_all(broker=broker)
print(term.clear_eos())
for stat in stats:
status = stat.status
# color status
if stat.status == Conf.WORKING:
status = term.green(str(Conf.WORKING))
elif stat.status == Conf.STOPPING:
status = term.yellow(str(Conf.STOPPING))
elif stat.status == Conf.STOPPED:
status = term.red(str(Conf.STOPPED))
elif stat.status == Conf.IDLE:
status = str(Conf.IDLE)
# color q's
tasks = str(stat.task_q_size)
if stat.task_q_size > 0:
tasks = term.cyan(str(stat.task_q_size))
if Conf.QUEUE_LIMIT and stat.task_q_size == Conf.QUEUE_LIMIT:
tasks = term.green(str(stat.task_q_size))
results = stat.done_q_size
if results > 0:
results = term.cyan(str(results))
# color workers
workers = len(stat.workers)
if workers < Conf.WORKERS:
workers = term.yellow(str(workers))
# format uptime
uptime = (timezone.now() - stat.tob).total_seconds()
hours, remainder = divmod(uptime, 3600)
minutes, seconds = divmod(remainder, 60)
uptime = "%d:%02d:%02d" % (hours, minutes, seconds)
# print to the terminal
print(
term.move(i, 0) +
term.center(stat.host[:col_width - 1], width=col_width -
1))
print(
term.move(i, 1 * col_width) +
term.center(str(stat.cluster_id)[-8:], width=col_width -
1))
print(
term.move(i, 2 * col_width) +
term.center(status, width=col_width - 1))
print(
term.move(i, 3 * col_width) +
term.center(workers, width=col_width - 1))
print(
term.move(i, 4 * col_width) +
term.center(tasks, width=col_width - 1))
print(
term.move(i, 5 * col_width) +
term.center(results, width=col_width - 1))
print(
term.move(i, 6 * col_width) +
term.center(stat.reincarnations, width=col_width - 1))
print(
term.move(i, 7 * col_width) +
term.center(uptime, width=col_width - 1))
i += 1
# bottom bar
i += 1
queue_size = broker.queue_size()
lock_size = broker.lock_size()
if lock_size:
queue_size = f"{queue_size}({lock_size})"
print(
term.move(i, 0) + term.white_on_cyan(
term.center(broker.info(), width=col_width * 2)))
print(
term.move(i, 2 * col_width) +
term.black_on_cyan(term.center(_("Queued"), width=col_width)))
print(
term.move(i, 3 * col_width) +
term.white_on_cyan(term.center(queue_size, width=col_width)))
print(
term.move(i, 4 * col_width) +
term.black_on_cyan(term.center(_("Success"), width=col_width)))
print(
term.move(i, 5 * col_width) + term.white_on_cyan(
term.center(models.Success.objects.count(),
width=col_width)))
print(
term.move(i, 6 * col_width) +
term.black_on_cyan(term.center(_("Failures"), width=col_width))
)
print(
term.move(i, 7 * col_width) + term.white_on_cyan(
term.center(models.Failure.objects.count(),
width=col_width)))
# for testing
if run_once:
return Stat.get_all(broker=broker)
print(term.move(i + 2, 0) + term.center(_("[Press q to quit]")))
val = term.inkey(timeout=1)
def memory(run_once=False, workers=False, broker=None):
if not broker:
broker = get_broker()
term = Terminal()
broker.ping()
if not psutil:
print(term.clear_eos())
print(
term.white_on_red(
'Cannot start "qmemory" command. Missing "psutil" library.'))
return
with term.fullscreen(), term.hidden_cursor(), term.cbreak():
MEMORY_AVAILABLE_LOWEST_PERCENTAGE = 100.0
MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
cols = 8
val = None
start_width = int(term.width / cols)
while val not in ["q", "Q"]:
col_width = int(term.width / cols)
# In case of resize
if col_width != start_width:
print(term.clear())
start_width = col_width
# sentinel, monitor and workers memory usage
print(
term.move(0, 0 * col_width) +
term.black_on_green(term.center(_("Host"), width=col_width -
1)))
print(
term.move(0, 1 * col_width) +
term.black_on_green(term.center(_("Id"), width=col_width - 1)))
print(
term.move(0, 2 * col_width) + term.black_on_green(
term.center(_("Available (%)"), width=col_width - 1)))
print(
term.move(0, 3 * col_width) + term.black_on_green(
term.center(_("Available (MB)"), width=col_width - 1)))
print(
term.move(0, 4 * col_width) + term.black_on_green(
term.center(_("Total (MB)"), width=col_width - 1)))
print(
term.move(0, 5 * col_width) + term.black_on_green(
term.center(_("Sentinel (MB)"), width=col_width - 1)))
print(
term.move(0, 6 * col_width) + term.black_on_green(
term.center(_("Monitor (MB)"), width=col_width - 1)))
print(
term.move(0, 7 * col_width) + term.black_on_green(
term.center(_("Workers (MB)"), width=col_width - 1)))
row = 2
stats = Stat.get_all(broker=broker)
print(term.clear_eos())
for stat in stats:
# memory available (%)
memory_available_percentage = round(
psutil.virtual_memory().available * 100 /
psutil.virtual_memory().total,
2,
)
# memory available (MB)
memory_available = round(
psutil.virtual_memory().available / 1024**2, 2)
if memory_available_percentage < MEMORY_AVAILABLE_LOWEST_PERCENTAGE:
MEMORY_AVAILABLE_LOWEST_PERCENTAGE = memory_available_percentage
MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
print(
term.move(row, 0 * col_width) +
term.center(stat.host[:col_width - 1], width=col_width -
1))
print(
term.move(row, 1 * col_width) +
term.center(str(stat.cluster_id)[-8:], width=col_width -
1))
print(
term.move(row, 2 * col_width) + term.center(
memory_available_percentage, width=col_width - 1))
print(
term.move(row, 3 * col_width) +
term.center(memory_available, width=col_width - 1))
print(
term.move(row, 4 * col_width) + term.center(
round(psutil.virtual_memory().total / 1024**2, 2),
width=col_width - 1,
))
print(
term.move(row, 5 * col_width) + term.center(
get_process_mb(stat.sentinel), width=col_width - 1))
print(
term.move(row, 6 * col_width) + term.center(
get_process_mb(getattr(stat, "monitor", None)),
width=col_width - 1,
))
workers_mb = 0
for worker_pid in stat.workers:
result = get_process_mb(worker_pid)
if isinstance(result, str):
result = 0
workers_mb += result
print(
term.move(row, 7 * col_width) +
term.center(workers_mb or "NO_PROCESSES_FOUND",
width=col_width - 1))
row += 1
# each worker's memory usage
if workers:
row += 2
col_width = int(term.width / (1 + Conf.WORKERS))
print(
term.move(row, 0 * col_width) + term.black_on_cyan(
term.center(_("Id"), width=col_width - 1)))
for worker_num in range(Conf.WORKERS):
print(
term.move(row, (worker_num + 1) * col_width) +
term.black_on_cyan(
term.center(
"Worker #{} (MB)".format(worker_num + 1),
width=col_width - 1,
)))
row += 2
for stat in stats:
print(
term.move(row, 0 * col_width) + term.center(
str(stat.cluster_id)[-8:], width=col_width - 1))
for idx, worker_pid in enumerate(stat.workers):
mb_used = get_process_mb(worker_pid)
print(
term.move(row, (idx + 1) * col_width) +
term.center(mb_used, width=col_width - 1))
row += 1
row += 1
print(
term.move(row, 0) + _("Available lowest (%): {} ({})").format(
str(MEMORY_AVAILABLE_LOWEST_PERCENTAGE),
MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT.strftime(
"%Y-%m-%d %H:%M:%S+00:00"),
))
# for testing
if run_once:
return Stat.get_all(broker=broker)
print(term.move(row + 2, 0) + term.center("[Press q to quit]"))
val = term.inkey(timeout=1)
def info(broker=None):
if not broker:
broker = get_broker()
term = Terminal()
broker.ping()
stat = Stat.get_all(broker=broker)
# general stats
clusters = len(stat)
workers = 0
reincarnations = 0
for cluster in stat:
workers += len(cluster.workers)
reincarnations += cluster.reincarnations
# calculate tasks pm and avg exec time
tasks_per = 0
per = _("day")
exec_time = 0
last_tasks = models.Success.objects.filter(stopped__gte=timezone.now() -
timedelta(hours=24))
tasks_per_day = last_tasks.count()
if tasks_per_day > 0:
# average execution time over the last 24 hours
if connection.vendor != "sqlite":
exec_time = last_tasks.aggregate(
time_taken=Sum(F("stopped") - F("started")))
exec_time = exec_time["time_taken"].total_seconds() / tasks_per_day
else:
# can't sum timedeltas on sqlite
for t in last_tasks:
exec_time += t.time_taken()
exec_time = exec_time / tasks_per_day
# tasks per second/minute/hour/day in the last 24 hours
if tasks_per_day > 24 * 60 * 60:
tasks_per = tasks_per_day / (24 * 60 * 60)
per = _("second")
elif tasks_per_day > 24 * 60:
tasks_per = tasks_per_day / (24 * 60)
per = _("minute")
elif tasks_per_day > 24:
tasks_per = tasks_per_day / 24
per = _("hour")
else:
tasks_per = tasks_per_day
# print to terminal
print(term.clear_eos())
col_width = int(term.width / 6)
print(
term.black_on_green(
term.center(
_(f'-- {Conf.PREFIX.capitalize()} { ".".join(str(v) for v in VERSION)} on {broker.info()} --'
))))
print(
term.cyan(_("Clusters")) + term.move_x(1 * col_width) +
term.white(str(clusters)) + term.move_x(2 * col_width) +
term.cyan(_("Workers")) + term.move_x(3 * col_width) +
term.white(str(workers)) + term.move_x(4 * col_width) +
term.cyan(_("Restarts")) + term.move_x(5 * col_width) +
term.white(str(reincarnations)))
print(
term.cyan(_("Queued")) + term.move_x(1 * col_width) +
term.white(str(broker.queue_size())) + term.move_x(2 * col_width) +
term.cyan(_("Successes")) + term.move_x(3 * col_width) +
term.white(str(models.Success.objects.count())) +
term.move_x(4 * col_width) + term.cyan(_("Failures")) +
term.move_x(5 * col_width) +
term.white(str(models.Failure.objects.count())))
print(
term.cyan(_("Schedules")) + term.move_x(1 * col_width) +
term.white(str(models.Schedule.objects.count())) +
term.move_x(2 * col_width) + term.cyan(_(f"Tasks/{per}")) +
term.move_x(3 * col_width) + term.white(f"{tasks_per:.2f}") +
term.move_x(4 * col_width) + term.cyan(_("Avg time")) +
term.move_x(5 * col_width) + term.white(f"{exec_time:.4f}"))
return True
def stat(self):
if self.sentinel:
return Stat.get(self.pid)
return Status(self.pid)
def stat(self):
if self.sentinel:
return Stat.get(self.pid)
return Status(self.pid)
def monitor(run_once=False, broker=None):
if not broker:
broker = get_broker()
term = Terminal()
broker.ping()
with term.fullscreen(), term.hidden_cursor(), term.cbreak():
val = None
start_width = int(term.width / 8)
while val not in (u'q', u'Q',):
col_width = int(term.width / 8)
# In case of resize
if col_width != start_width:
print(term.clear())
start_width = col_width
print(term.move(0, 0) + term.black_on_green(term.center(_('Host'), width=col_width - 1)))
print(term.move(0, 1 * col_width) + term.black_on_green(term.center(_('Id'), width=col_width - 1)))
print(term.move(0, 2 * col_width) + term.black_on_green(term.center(_('State'), width=col_width - 1)))
print(term.move(0, 3 * col_width) + term.black_on_green(term.center(_('Pool'), width=col_width - 1)))
print(term.move(0, 4 * col_width) + term.black_on_green(term.center(_('TQ'), width=col_width - 1)))
print(term.move(0, 5 * col_width) + term.black_on_green(term.center(_('RQ'), width=col_width - 1)))
print(term.move(0, 6 * col_width) + term.black_on_green(term.center(_('RC'), width=col_width - 1)))
print(term.move(0, 7 * col_width) + term.black_on_green(term.center(_('Up'), width=col_width - 1)))
i = 2
stats = Stat.get_all(broker=broker)
print(term.clear_eos())
for stat in stats:
status = stat.status
# color status
if stat.status == Conf.WORKING:
status = term.green(str(Conf.WORKING))
elif stat.status == Conf.STOPPING:
status = term.yellow(str(Conf.STOPPING))
elif stat.status == Conf.STOPPED:
status = term.red(str(Conf.STOPPED))
elif stat.status == Conf.IDLE:
status = str(Conf.IDLE)
# color q's
tasks = str(stat.task_q_size)
if stat.task_q_size > 0:
tasks = term.cyan(str(stat.task_q_size))
if Conf.QUEUE_LIMIT and stat.task_q_size == Conf.QUEUE_LIMIT:
tasks = term.green(str(stat.task_q_size))
results = stat.done_q_size
if results > 0:
results = term.cyan(str(results))
# color workers
workers = len(stat.workers)
if workers < Conf.WORKERS:
workers = term.yellow(str(workers))
# format uptime
uptime = (timezone.now() - stat.tob).total_seconds()
hours, remainder = divmod(uptime, 3600)
minutes, seconds = divmod(remainder, 60)
uptime = '%d:%02d:%02d' % (hours, minutes, seconds)
# print to the terminal
print(term.move(i, 0) + term.center(stat.host[:col_width - 1], width=col_width - 1))
print(term.move(i, 1 * col_width) + term.center(stat.cluster_id, width=col_width - 1))
print(term.move(i, 2 * col_width) + term.center(status, width=col_width - 1))
print(term.move(i, 3 * col_width) + term.center(workers, width=col_width - 1))
print(term.move(i, 4 * col_width) + term.center(tasks, width=col_width - 1))
print(term.move(i, 5 * col_width) + term.center(results, width=col_width - 1))
print(term.move(i, 6 * col_width) + term.center(stat.reincarnations, width=col_width - 1))
print(term.move(i, 7 * col_width) + term.center(uptime, width=col_width - 1))
i += 1
# bottom bar
i += 1
queue_size = broker.queue_size()
lock_size = broker.lock_size()
if lock_size:
queue_size = '{}({})'.format(queue_size, lock_size)
print(term.move(i, 0) + term.white_on_cyan(term.center(broker.info(), width=col_width * 2)))
print(term.move(i, 2 * col_width) + term.black_on_cyan(term.center(_('Queued'), width=col_width)))
print(term.move(i, 3 * col_width) + term.white_on_cyan(term.center(queue_size, width=col_width)))
print(term.move(i, 4 * col_width) + term.black_on_cyan(term.center(_('Success'), width=col_width)))
print(term.move(i, 5 * col_width) + term.white_on_cyan(
term.center(models.Success.objects.count(), width=col_width)))
print(term.move(i, 6 * col_width) + term.black_on_cyan(term.center(_('Failures'), width=col_width)))
print(term.move(i, 7 * col_width) + term.white_on_cyan(
term.center(models.Failure.objects.count(), width=col_width)))
# for testing
if run_once:
return Stat.get_all(broker=broker)
print(term.move(i + 2, 0) + term.center(_('[Press q to quit]')))
val = term.inkey(timeout=1)
def info(broker=None):
if not broker:
broker = get_broker()
term = Terminal()
broker.ping()
stat = Stat.get_all(broker=broker)
# general stats
clusters = len(stat)
workers = 0
reincarnations = 0
for cluster in stat:
workers += len(cluster.workers)
reincarnations += cluster.reincarnations
# calculate tasks pm and avg exec time
tasks_per = 0
per = _('day')
exec_time = 0
last_tasks = models.Success.objects.filter(stopped__gte=timezone.now() - timedelta(hours=24))
tasks_per_day = last_tasks.count()
if tasks_per_day > 0:
# average execution time over the last 24 hours
if not connection.vendor == 'sqlite':
exec_time = last_tasks.aggregate(time_taken=Sum(F('stopped') - F('started')))
exec_time = exec_time['time_taken'].total_seconds() / tasks_per_day
else:
# can't sum timedeltas on sqlite
for t in last_tasks:
exec_time += t.time_taken()
exec_time = exec_time / tasks_per_day
# tasks per second/minute/hour/day in the last 24 hours
if tasks_per_day > 24 * 60 * 60:
tasks_per = tasks_per_day / (24 * 60 * 60)
per = _('second')
elif tasks_per_day > 24 * 60:
tasks_per = tasks_per_day / (24 * 60)
per = _('minute')
elif tasks_per_day > 24:
tasks_per = tasks_per_day / 24
per = _('hour')
else:
tasks_per = tasks_per_day
# print to terminal
print(term.clear_eos())
col_width = int(term.width / 6)
print(term.black_on_green(
term.center(
_('-- {} {} on {} --').format(Conf.PREFIX.capitalize(), '.'.join(str(v) for v in VERSION),
broker.info()))))
print(term.cyan(_('Clusters')) +
term.move_x(1 * col_width) +
term.white(str(clusters)) +
term.move_x(2 * col_width) +
term.cyan(_('Workers')) +
term.move_x(3 * col_width) +
term.white(str(workers)) +
term.move_x(4 * col_width) +
term.cyan(_('Restarts')) +
term.move_x(5 * col_width) +
term.white(str(reincarnations))
)
print(term.cyan(_('Queued')) +
term.move_x(1 * col_width) +
term.white(str(broker.queue_size())) +
term.move_x(2 * col_width) +
term.cyan(_('Successes')) +
term.move_x(3 * col_width) +
term.white(str(models.Success.objects.count())) +
term.move_x(4 * col_width) +
term.cyan(_('Failures')) +
term.move_x(5 * col_width) +
term.white(str(models.Failure.objects.count()))
)
print(term.cyan(_('Schedules')) +
term.move_x(1 * col_width) +
term.white(str(models.Schedule.objects.count())) +
term.move_x(2 * col_width) +
term.cyan(_('Tasks/{}'.format(per))) +
term.move_x(3 * col_width) +
term.white('{0:.2f}'.format(tasks_per)) +
term.move_x(4 * col_width) +
term.cyan(_('Avg time')) +
term.move_x(5 * col_width) +
term.white('{0:.4f}'.format(exec_time))
)
return True
def info(broker=None):
if not broker:
broker = get_broker()
term = Terminal()
broker.ping()
stat = Stat.get_all(broker=broker)
# general stats
clusters = len(stat)
workers = 0
reincarnations = 0
for cluster in stat:
workers += len(cluster.workers)
reincarnations += cluster.reincarnations
# calculate tasks pm and avg exec time
tasks_per = 0
per = _('day')
exec_time = 0
last_tasks = models.Success.objects.filter(stopped__gte=timezone.now() -
timedelta(hours=24))
tasks_per_day = last_tasks.count()
if tasks_per_day > 0:
# average execution time over the last 24 hours
if not connection.vendor == 'sqlite':
exec_time = last_tasks.aggregate(
time_taken=Sum(F('stopped') - F('started')))
exec_time = exec_time['time_taken'].total_seconds() / tasks_per_day
else:
# can't sum timedeltas on sqlite
for t in last_tasks:
exec_time += t.time_taken()
exec_time = exec_time / tasks_per_day
# tasks per second/minute/hour/day in the last 24 hours
if tasks_per_day > 24 * 60 * 60:
tasks_per = tasks_per_day / (24 * 60 * 60)
per = _('second')
elif tasks_per_day > 24 * 60:
tasks_per = tasks_per_day / (24 * 60)
per = _('minute')
elif tasks_per_day > 24:
tasks_per = tasks_per_day / 24
per = _('hour')
else:
tasks_per = tasks_per_day
# print to terminal
print(term.clear_eos())
col_width = int(term.width / 6)
print(
term.black_on_green(
term.center(
_('-- {} {} on {} --').format(
Conf.PREFIX.capitalize(),
'.'.join(str(v) for v in VERSION), broker.info()))))
print(
term.cyan(_('Clusters')) + term.move_x(1 * col_width) +
term.white(str(clusters)) + term.move_x(2 * col_width) +
term.cyan(_('Workers')) + term.move_x(3 * col_width) +
term.white(str(workers)) + term.move_x(4 * col_width) +
term.cyan(_('Restarts')) + term.move_x(5 * col_width) +
term.white(str(reincarnations)))
print(
term.cyan(_('Queued')) + term.move_x(1 * col_width) +
term.white(str(broker.queue_size())) + term.move_x(2 * col_width) +
term.cyan(_('Successes')) + term.move_x(3 * col_width) +
term.white(str(models.Success.objects.count())) +
term.move_x(4 * col_width) + term.cyan(_('Failures')) +
term.move_x(5 * col_width) +
term.white(str(models.Failure.objects.count())))
print(
term.cyan(_('Schedules')) + term.move_x(1 * col_width) +
term.white(str(models.Schedule.objects.count())) +
term.move_x(2 * col_width) + term.cyan(_('Tasks/{}'.format(per))) +
term.move_x(3 * col_width) + term.white('{0:.2f}'.format(tasks_per)) +
term.move_x(4 * col_width) + term.cyan(_('Avg time')) +
term.move_x(5 * col_width) + term.white('{0:.4f}'.format(exec_time)))
return True
def debug(request):
# 获取完整信息
full = request.GET.get('full')
# 系统配置
sys_config = SysConfig().sys_config
# MySQL信息
cursor = connection.cursor()
mysql_info = {
'mysql_server_info': cursor.db.mysql_server_info,
'timezone_name': cursor.db.timezone_name
}
# Redis信息
redis_conn = get_redis_connection("default")
full_redis_info = redis_conn.info()
redis_info = {
'redis_version': full_redis_info.get('redis_version'),
'redis_mode': full_redis_info.get('redis_mode'),
'role': full_redis_info.get('role'),
'maxmemory_human': full_redis_info.get('maxmemory_human'),
'used_memory_human': full_redis_info.get('used_memory_human'),
}
# django_q
django_q_version = '.'.join(str(i) for i in django_q.VERSION)
broker = get_broker()
stats = Stat.get_all(broker=broker)
queue_size = broker.queue_size()
lock_size = broker.lock_size()
if lock_size:
queue_size = '{}({})'.format(queue_size, lock_size)
q_broker_stats = {
'info': broker.info(),
'Queued': queue_size,
'Success': Success.objects.count(),
'Failures': Failure.objects.count(),
}
q_cluster_stats = []
for stat in stats:
# format uptime
uptime = (timezone.now() - stat.tob).total_seconds()
hours, remainder = divmod(uptime, 3600)
minutes, seconds = divmod(remainder, 60)
uptime = '%d:%02d:%02d' % (hours, minutes, seconds)
q_cluster_stats.append({
'host': stat.host,
'cluster_id': stat.cluster_id,
'state': stat.status,
'pool': len(stat.workers),
'tq': stat.task_q_size,
'rq': stat.done_q_size,
'rc': stat.reincarnations,
'up': uptime
})
# Inception和goInception信息
inception_host = sys_config.get('inception_host')
inception_port = sys_config.get('inception_port', 0)
go_inception_host = sys_config.get('go_inception_host')
go_inception_port = sys_config.get('go_inception_port', 0)
inception_remote_backup_host = sys_config.get(
'inception_remote_backup_host', '')
inception_remote_backup_port = sys_config.get(
'inception_remote_backup_port', '')
inception_remote_backup_user = sys_config.get(
'inception_remote_backup_user', '')
inception_remote_backup_password = sys_config.get(
'inception_remote_backup_password', '')
# inception
try:
inc_conn = MySQLdb.connect(host=inception_host,
port=int(inception_port),
connect_timeout=1,
cursorclass=MySQLdb.cursors.DictCursor)
cursor = inc_conn.cursor()
cursor.execute('inception get variables')
rows = cursor.fetchall()
full_inception_info = dict()
for row in rows:
full_inception_info[row.get('Variable_name')] = row.get('Value')
inception_info = {
'version':
full_inception_info.get('version'),
'max_allowed_packet':
full_inception_info.get('max_allowed_packet'),
'inception_language_code':
full_inception_info.get('inception_language_code'),
'inception_osc_on':
full_inception_info.get('inception_osc_on'),
'inception_osc_bin_dir':
full_inception_info.get('inception_osc_bin_dir'),
}
except Exception as e:
inception_info = f'获取Inception信息报错:{e}'
full_inception_info = inception_info
# goInception
try:
goinc_conn = MySQLdb.connect(host=go_inception_host,
port=int(go_inception_port),
connect_timeout=1,
cursorclass=MySQLdb.cursors.DictCursor)
cursor = goinc_conn.cursor()
cursor.execute('inception get variables')
rows = cursor.fetchall()
full_goinception_info = dict()
for row in rows:
full_goinception_info[row.get('Variable_name')] = row.get('Value')
goinception_info = {
'version': full_goinception_info.get('version'),
'max_allowed_packet':
full_goinception_info.get('max_allowed_packet'),
'lang': full_goinception_info.get('lang'),
'osc_on': full_goinception_info.get('osc_on'),
'osc_bin_dir': full_goinception_info.get('osc_bin_dir'),
'ghost_on': full_goinception_info.get('ghost_on'),
}
except Exception as e:
goinception_info = f'获取goInception信息报错:{e}'
full_goinception_info = goinception_info
# 备份库
try:
bak_conn = MySQLdb.connect(host=inception_remote_backup_host,
port=int(inception_remote_backup_port),
user=inception_remote_backup_user,
password=inception_remote_backup_password,
connect_timeout=1)
cursor = bak_conn.cursor()
cursor.execute('select 1;')
backup_info = 'normal'
except Exception as e:
backup_info = f'无法连接Inception备份库\n{e}'
# PACKAGES
installed_packages = pkg_resources.working_set
installed_packages_list = sorted(
["%s==%s" % (i.key, i.version) for i in installed_packages])
# 最终集合
system_info = {
'archery': {
'version': archery.display_version
},
'django_q': {
'version': django_q_version,
'conf': django_q.conf.Conf.conf,
'q_cluster_stats': q_cluster_stats
if q_cluster_stats else '没有正在运行的集群信息，请检查django_q状态',
'q_broker_stats': q_broker_stats
},
'inception': {
'enable_inception': sys_config.get('inception'),
'inception_info': full_inception_info if full else inception_info,
'goinception_info':
full_goinception_info if full else goinception_info,
'backup_info': backup_info
},
'runtime_info': {
'python_version': platform.python_version(),
'mysql_info': mysql_info,
'redis_info': full_redis_info if full else redis_info,
'sys_argv': sys.argv,
'platform': platform.uname()
},
'sys_config': sys_config,
'packages': installed_packages_list
}
return JsonResponse(system_info)
def stat(self):
if self.sentinel:
return Stat.get(pid=self.pid, cluster_id=self.cluster_id)
return Status(pid=self.pid, cluster_id=self.cluster_id)
