def get_ids(): # prints id (PID) of running clusters stat = Stat.get_all() if stat: for s in stat: print(s.cluster_id) else: print('No clusters appear to be running.') return True
def test_bad_secret(broker, monkeypatch): broker.list_key = 'test_bad_secret:q' async('math.copysign', 1, -1, broker=broker) stop_event = Event() stop_event.set() start_event = Event() s = Sentinel(stop_event, start_event, broker=broker, start=False) Stat(s).save() # change the SECRET monkeypatch.setattr(Conf, "SECRET_KEY", "OOPS") stat = Stat.get_all() assert len(stat) == 0 assert Stat.get(s.parent_pid) is None task_queue = Queue() pusher(task_queue, stop_event, broker=broker) result_queue = Queue() task_queue.put('STOP') worker(task_queue, result_queue, Value('f', -1), ) assert result_queue.qsize() == 0 broker.delete_queue()
def stop(self): Stat(self).save() name = current_process().name logger.info(_(f"{name} stopping cluster processes")) # Stopping pusher self.event_out.set() # Wait for it to stop while self.pusher.is_alive(): sleep(0.1) Stat(self).save() # Put poison pills in the queue for __ in range(len(self.pool)): self.task_queue.put("STOP") self.task_queue.close() # wait for the task queue to empty self.task_queue.join_thread() # Wait for all the workers to exit while len(self.pool): for p in self.pool: if not p.is_alive(): self.pool.remove(p) sleep(0.1) Stat(self).save() # Finally stop the monitor self.result_queue.put("STOP") self.result_queue.close() # Wait for the result queue to empty self.result_queue.join_thread() logger.info(_(f"{name} waiting for the monitor.")) # Wait for everything to close or time out count = 0 if not self.timeout: self.timeout = 30 while self.status() == Conf.STOPPING and count < self.timeout * 10: sleep(0.1) Stat(self).save() count += 1 # Final status Stat(self).save()
def spawn_cluster(self): self.pool = [] Stat(self).save() close_old_django_connections() # spawn worker pool for __ in range(self.pool_size): self.spawn_worker() # spawn auxiliary self.monitor = self.spawn_monitor() self.pusher = self.spawn_pusher() # set worker cpu affinity if needed if psutil and Conf.CPU_AFFINITY: set_cpu_affinity(Conf.CPU_AFFINITY, [w.pid for w in self.pool])
def guard(self): logger.info( _( f"{current_process().name} guarding cluster {humanize(self.cluster_id.hex)}" ) ) self.start_event.set() Stat(self).save() logger.info(_(f"Q Cluster {humanize(self.cluster_id.hex)} running.")) counter = 0 cycle = Conf.GUARD_CYCLE # guard loop sleep in seconds # Guard loop. Runs at least once while not self.stop_event.is_set() or not counter: # Check Workers for p in self.pool: with p.timer.get_lock(): # Are you alive? if not p.is_alive() or p.timer.value == 0: self.reincarnate(p) continue # Decrement timer if work is being done if p.timer.value > 0: p.timer.value -= cycle # Check Monitor if not self.monitor.is_alive(): self.reincarnate(self.monitor) # Check Pusher if not self.pusher.is_alive(): self.reincarnate(self.pusher) # Call scheduler once a minute (or so) counter += cycle if counter >= 30 and Conf.SCHEDULER: counter = 0 scheduler(broker=self.broker) # Save current status Stat(self).save() sleep(cycle) self.stop()
def guard(self): logger.info( _('{} guarding cluster at {}').format(current_process().name, self.pid)) self.start_event.set() Stat(self).save() logger.info(_('Q Cluster-{} running.').format(self.parent_pid)) scheduler(broker=self.broker) counter = 0 cycle = 0.5 # guard loop sleep in seconds # Guard loop. Runs at least once while not self.stop_event.is_set() or not counter: # Check Workers for p in self.pool: # Are you alive? if not p.is_alive() or (self.timeout and p.timer.value == 0): self.reincarnate(p) continue # Decrement timer if work is being done if self.timeout and p.timer.value > 0: p.timer.value -= cycle # Check Monitor if not self.monitor.is_alive(): self.reincarnate(self.monitor) # Check Pusher if not self.pusher.is_alive(): self.reincarnate(self.pusher) # Call scheduler once a minute (or so) counter += cycle if counter == 30 and Conf.SCHEDULER: counter = 0 scheduler(broker=self.broker) # Save current status Stat(self).save() sleep(cycle) self.stop()
def test_bad_secret(broker, monkeypatch): broker.list_key = 'test_bad_secret:q' async ('math.copysign', 1, -1, broker=broker) stop_event = Event() stop_event.set() start_event = Event() s = Sentinel(stop_event, start_event, broker=broker, start=False) Stat(s).save() # change the SECRET monkeypatch.setattr(Conf, "SECRET_KEY", "OOPS") stat = Stat.get_all() assert len(stat) == 0 assert Stat.get(s.parent_pid) is None task_queue = Queue() pusher(task_queue, stop_event, broker=broker) result_queue = Queue() task_queue.put('STOP') worker( task_queue, result_queue, Value('f', -1), ) assert result_queue.qsize() == 0 broker.delete_queue()
def test_monitor(monkeypatch): assert Stat.get(0).sentinel == 0 c = Cluster() c.start() stats = monitor(run_once=True) c.stop() assert len(stats) > 0 found_c = False for stat in stats: if stat.cluster_id == c.pid: found_c = True assert stat.uptime() > 0 assert stat.empty_queues() is True break assert found_c is True # test lock size monkeypatch.setattr(Conf, 'ORM', 'default') b = get_broker('monitor_test') b.enqueue('test') b.dequeue() assert b.lock_size() == 1 monitor(run_once=True, broker=b) b.delete_queue()
def monitor(run_once=False, broker=None): if not broker: broker = get_broker() term = Terminal() broker.ping() with term.fullscreen(), term.hidden_cursor(), term.cbreak(): val = None start_width = int(term.width / 8) while val not in ( "q", "Q", ): col_width = int(term.width / 8) # In case of resize if col_width != start_width: print(term.clear()) start_width = col_width print( term.move(0, 0) + term.black_on_green(term.center(_("Host"), width=col_width - 1))) print( term.move(0, 1 * col_width) + term.black_on_green(term.center(_("Id"), width=col_width - 1))) print( term.move(0, 2 * col_width) + term.black_on_green( term.center(_("State"), width=col_width - 1))) print( term.move(0, 3 * col_width) + term.black_on_green(term.center(_("Pool"), width=col_width - 1))) print( term.move(0, 4 * col_width) + term.black_on_green(term.center(_("TQ"), width=col_width - 1))) print( term.move(0, 5 * col_width) + term.black_on_green(term.center(_("RQ"), width=col_width - 1))) print( term.move(0, 6 * col_width) + term.black_on_green(term.center(_("RC"), width=col_width - 1))) print( term.move(0, 7 * col_width) + term.black_on_green(term.center(_("Up"), width=col_width - 1))) i = 2 stats = Stat.get_all(broker=broker) print(term.clear_eos()) for stat in stats: status = stat.status # color status if stat.status == Conf.WORKING: status = term.green(str(Conf.WORKING)) elif stat.status == Conf.STOPPING: status = term.yellow(str(Conf.STOPPING)) elif stat.status == Conf.STOPPED: status = term.red(str(Conf.STOPPED)) elif stat.status == Conf.IDLE: status = str(Conf.IDLE) # color q's tasks = str(stat.task_q_size) if stat.task_q_size > 0: tasks = term.cyan(str(stat.task_q_size)) if Conf.QUEUE_LIMIT and stat.task_q_size == Conf.QUEUE_LIMIT: tasks = term.green(str(stat.task_q_size)) results = stat.done_q_size if results > 0: results = term.cyan(str(results)) # color workers workers = len(stat.workers) if workers < Conf.WORKERS: workers = term.yellow(str(workers)) # format uptime uptime = (timezone.now() - stat.tob).total_seconds() hours, remainder = divmod(uptime, 3600) minutes, seconds = divmod(remainder, 60) uptime = "%d:%02d:%02d" % (hours, minutes, seconds) # print to the terminal print( term.move(i, 0) + term.center(stat.host[:col_width - 1], width=col_width - 1)) print( term.move(i, 1 * col_width) + term.center(str(stat.cluster_id)[-8:], width=col_width - 1)) print( term.move(i, 2 * col_width) + term.center(status, width=col_width - 1)) print( term.move(i, 3 * col_width) + term.center(workers, width=col_width - 1)) print( term.move(i, 4 * col_width) + term.center(tasks, width=col_width - 1)) print( term.move(i, 5 * col_width) + term.center(results, width=col_width - 1)) print( term.move(i, 6 * col_width) + term.center(stat.reincarnations, width=col_width - 1)) print( term.move(i, 7 * col_width) + term.center(uptime, width=col_width - 1)) i += 1 # bottom bar i += 1 queue_size = broker.queue_size() lock_size = broker.lock_size() if lock_size: queue_size = f"{queue_size}({lock_size})" print( term.move(i, 0) + term.white_on_cyan( term.center(broker.info(), width=col_width * 2))) print( term.move(i, 2 * col_width) + term.black_on_cyan(term.center(_("Queued"), width=col_width))) print( term.move(i, 3 * col_width) + term.white_on_cyan(term.center(queue_size, width=col_width))) print( term.move(i, 4 * col_width) + term.black_on_cyan(term.center(_("Success"), width=col_width))) print( term.move(i, 5 * col_width) + term.white_on_cyan( term.center(models.Success.objects.count(), width=col_width))) print( term.move(i, 6 * col_width) + term.black_on_cyan(term.center(_("Failures"), width=col_width)) ) print( term.move(i, 7 * col_width) + term.white_on_cyan( term.center(models.Failure.objects.count(), width=col_width))) # for testing if run_once: return Stat.get_all(broker=broker) print(term.move(i + 2, 0) + term.center(_("[Press q to quit]"))) val = term.inkey(timeout=1)
def memory(run_once=False, workers=False, broker=None): if not broker: broker = get_broker() term = Terminal() broker.ping() if not psutil: print(term.clear_eos()) print( term.white_on_red( 'Cannot start "qmemory" command. Missing "psutil" library.')) return with term.fullscreen(), term.hidden_cursor(), term.cbreak(): MEMORY_AVAILABLE_LOWEST_PERCENTAGE = 100.0 MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now() cols = 8 val = None start_width = int(term.width / cols) while val not in ["q", "Q"]: col_width = int(term.width / cols) # In case of resize if col_width != start_width: print(term.clear()) start_width = col_width # sentinel, monitor and workers memory usage print( term.move(0, 0 * col_width) + term.black_on_green(term.center(_("Host"), width=col_width - 1))) print( term.move(0, 1 * col_width) + term.black_on_green(term.center(_("Id"), width=col_width - 1))) print( term.move(0, 2 * col_width) + term.black_on_green( term.center(_("Available (%)"), width=col_width - 1))) print( term.move(0, 3 * col_width) + term.black_on_green( term.center(_("Available (MB)"), width=col_width - 1))) print( term.move(0, 4 * col_width) + term.black_on_green( term.center(_("Total (MB)"), width=col_width - 1))) print( term.move(0, 5 * col_width) + term.black_on_green( term.center(_("Sentinel (MB)"), width=col_width - 1))) print( term.move(0, 6 * col_width) + term.black_on_green( term.center(_("Monitor (MB)"), width=col_width - 1))) print( term.move(0, 7 * col_width) + term.black_on_green( term.center(_("Workers (MB)"), width=col_width - 1))) row = 2 stats = Stat.get_all(broker=broker) print(term.clear_eos()) for stat in stats: # memory available (%) memory_available_percentage = round( psutil.virtual_memory().available * 100 / psutil.virtual_memory().total, 2, ) # memory available (MB) memory_available = round( psutil.virtual_memory().available / 1024**2, 2) if memory_available_percentage < MEMORY_AVAILABLE_LOWEST_PERCENTAGE: MEMORY_AVAILABLE_LOWEST_PERCENTAGE = memory_available_percentage MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now() print( term.move(row, 0 * col_width) + term.center(stat.host[:col_width - 1], width=col_width - 1)) print( term.move(row, 1 * col_width) + term.center(str(stat.cluster_id)[-8:], width=col_width - 1)) print( term.move(row, 2 * col_width) + term.center( memory_available_percentage, width=col_width - 1)) print( term.move(row, 3 * col_width) + term.center(memory_available, width=col_width - 1)) print( term.move(row, 4 * col_width) + term.center( round(psutil.virtual_memory().total / 1024**2, 2), width=col_width - 1, )) print( term.move(row, 5 * col_width) + term.center( get_process_mb(stat.sentinel), width=col_width - 1)) print( term.move(row, 6 * col_width) + term.center( get_process_mb(getattr(stat, "monitor", None)), width=col_width - 1, )) workers_mb = 0 for worker_pid in stat.workers: result = get_process_mb(worker_pid) if isinstance(result, str): result = 0 workers_mb += result print( term.move(row, 7 * col_width) + term.center(workers_mb or "NO_PROCESSES_FOUND", width=col_width - 1)) row += 1 # each worker's memory usage if workers: row += 2 col_width = int(term.width / (1 + Conf.WORKERS)) print( term.move(row, 0 * col_width) + term.black_on_cyan( term.center(_("Id"), width=col_width - 1))) for worker_num in range(Conf.WORKERS): print( term.move(row, (worker_num + 1) * col_width) + term.black_on_cyan( term.center( "Worker #{} (MB)".format(worker_num + 1), width=col_width - 1, ))) row += 2 for stat in stats: print( term.move(row, 0 * col_width) + term.center( str(stat.cluster_id)[-8:], width=col_width - 1)) for idx, worker_pid in enumerate(stat.workers): mb_used = get_process_mb(worker_pid) print( term.move(row, (idx + 1) * col_width) + term.center(mb_used, width=col_width - 1)) row += 1 row += 1 print( term.move(row, 0) + _("Available lowest (%): {} ({})").format( str(MEMORY_AVAILABLE_LOWEST_PERCENTAGE), MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT.strftime( "%Y-%m-%d %H:%M:%S+00:00"), )) # for testing if run_once: return Stat.get_all(broker=broker) print(term.move(row + 2, 0) + term.center("[Press q to quit]")) val = term.inkey(timeout=1)
def info(broker=None): if not broker: broker = get_broker() term = Terminal() broker.ping() stat = Stat.get_all(broker=broker) # general stats clusters = len(stat) workers = 0 reincarnations = 0 for cluster in stat: workers += len(cluster.workers) reincarnations += cluster.reincarnations # calculate tasks pm and avg exec time tasks_per = 0 per = _("day") exec_time = 0 last_tasks = models.Success.objects.filter(stopped__gte=timezone.now() - timedelta(hours=24)) tasks_per_day = last_tasks.count() if tasks_per_day > 0: # average execution time over the last 24 hours if connection.vendor != "sqlite": exec_time = last_tasks.aggregate( time_taken=Sum(F("stopped") - F("started"))) exec_time = exec_time["time_taken"].total_seconds() / tasks_per_day else: # can't sum timedeltas on sqlite for t in last_tasks: exec_time += t.time_taken() exec_time = exec_time / tasks_per_day # tasks per second/minute/hour/day in the last 24 hours if tasks_per_day > 24 * 60 * 60: tasks_per = tasks_per_day / (24 * 60 * 60) per = _("second") elif tasks_per_day > 24 * 60: tasks_per = tasks_per_day / (24 * 60) per = _("minute") elif tasks_per_day > 24: tasks_per = tasks_per_day / 24 per = _("hour") else: tasks_per = tasks_per_day # print to terminal print(term.clear_eos()) col_width = int(term.width / 6) print( term.black_on_green( term.center( _(f'-- {Conf.PREFIX.capitalize()} { ".".join(str(v) for v in VERSION)} on {broker.info()} --' )))) print( term.cyan(_("Clusters")) + term.move_x(1 * col_width) + term.white(str(clusters)) + term.move_x(2 * col_width) + term.cyan(_("Workers")) + term.move_x(3 * col_width) + term.white(str(workers)) + term.move_x(4 * col_width) + term.cyan(_("Restarts")) + term.move_x(5 * col_width) + term.white(str(reincarnations))) print( term.cyan(_("Queued")) + term.move_x(1 * col_width) + term.white(str(broker.queue_size())) + term.move_x(2 * col_width) + term.cyan(_("Successes")) + term.move_x(3 * col_width) + term.white(str(models.Success.objects.count())) + term.move_x(4 * col_width) + term.cyan(_("Failures")) + term.move_x(5 * col_width) + term.white(str(models.Failure.objects.count()))) print( term.cyan(_("Schedules")) + term.move_x(1 * col_width) + term.white(str(models.Schedule.objects.count())) + term.move_x(2 * col_width) + term.cyan(_(f"Tasks/{per}")) + term.move_x(3 * col_width) + term.white(f"{tasks_per:.2f}") + term.move_x(4 * col_width) + term.cyan(_("Avg time")) + term.move_x(5 * col_width) + term.white(f"{exec_time:.4f}")) return True
def stat(self): if self.sentinel: return Stat.get(self.pid) return Status(self.pid)
def monitor(run_once=False, broker=None): if not broker: broker = get_broker() term = Terminal() broker.ping() with term.fullscreen(), term.hidden_cursor(), term.cbreak(): val = None start_width = int(term.width / 8) while val not in (u'q', u'Q',): col_width = int(term.width / 8) # In case of resize if col_width != start_width: print(term.clear()) start_width = col_width print(term.move(0, 0) + term.black_on_green(term.center(_('Host'), width=col_width - 1))) print(term.move(0, 1 * col_width) + term.black_on_green(term.center(_('Id'), width=col_width - 1))) print(term.move(0, 2 * col_width) + term.black_on_green(term.center(_('State'), width=col_width - 1))) print(term.move(0, 3 * col_width) + term.black_on_green(term.center(_('Pool'), width=col_width - 1))) print(term.move(0, 4 * col_width) + term.black_on_green(term.center(_('TQ'), width=col_width - 1))) print(term.move(0, 5 * col_width) + term.black_on_green(term.center(_('RQ'), width=col_width - 1))) print(term.move(0, 6 * col_width) + term.black_on_green(term.center(_('RC'), width=col_width - 1))) print(term.move(0, 7 * col_width) + term.black_on_green(term.center(_('Up'), width=col_width - 1))) i = 2 stats = Stat.get_all(broker=broker) print(term.clear_eos()) for stat in stats: status = stat.status # color status if stat.status == Conf.WORKING: status = term.green(str(Conf.WORKING)) elif stat.status == Conf.STOPPING: status = term.yellow(str(Conf.STOPPING)) elif stat.status == Conf.STOPPED: status = term.red(str(Conf.STOPPED)) elif stat.status == Conf.IDLE: status = str(Conf.IDLE) # color q's tasks = str(stat.task_q_size) if stat.task_q_size > 0: tasks = term.cyan(str(stat.task_q_size)) if Conf.QUEUE_LIMIT and stat.task_q_size == Conf.QUEUE_LIMIT: tasks = term.green(str(stat.task_q_size)) results = stat.done_q_size if results > 0: results = term.cyan(str(results)) # color workers workers = len(stat.workers) if workers < Conf.WORKERS: workers = term.yellow(str(workers)) # format uptime uptime = (timezone.now() - stat.tob).total_seconds() hours, remainder = divmod(uptime, 3600) minutes, seconds = divmod(remainder, 60) uptime = '%d:%02d:%02d' % (hours, minutes, seconds) # print to the terminal print(term.move(i, 0) + term.center(stat.host[:col_width - 1], width=col_width - 1)) print(term.move(i, 1 * col_width) + term.center(stat.cluster_id, width=col_width - 1)) print(term.move(i, 2 * col_width) + term.center(status, width=col_width - 1)) print(term.move(i, 3 * col_width) + term.center(workers, width=col_width - 1)) print(term.move(i, 4 * col_width) + term.center(tasks, width=col_width - 1)) print(term.move(i, 5 * col_width) + term.center(results, width=col_width - 1)) print(term.move(i, 6 * col_width) + term.center(stat.reincarnations, width=col_width - 1)) print(term.move(i, 7 * col_width) + term.center(uptime, width=col_width - 1)) i += 1 # bottom bar i += 1 queue_size = broker.queue_size() lock_size = broker.lock_size() if lock_size: queue_size = '{}({})'.format(queue_size, lock_size) print(term.move(i, 0) + term.white_on_cyan(term.center(broker.info(), width=col_width * 2))) print(term.move(i, 2 * col_width) + term.black_on_cyan(term.center(_('Queued'), width=col_width))) print(term.move(i, 3 * col_width) + term.white_on_cyan(term.center(queue_size, width=col_width))) print(term.move(i, 4 * col_width) + term.black_on_cyan(term.center(_('Success'), width=col_width))) print(term.move(i, 5 * col_width) + term.white_on_cyan( term.center(models.Success.objects.count(), width=col_width))) print(term.move(i, 6 * col_width) + term.black_on_cyan(term.center(_('Failures'), width=col_width))) print(term.move(i, 7 * col_width) + term.white_on_cyan( term.center(models.Failure.objects.count(), width=col_width))) # for testing if run_once: return Stat.get_all(broker=broker) print(term.move(i + 2, 0) + term.center(_('[Press q to quit]'))) val = term.inkey(timeout=1)
def info(broker=None): if not broker: broker = get_broker() term = Terminal() broker.ping() stat = Stat.get_all(broker=broker) # general stats clusters = len(stat) workers = 0 reincarnations = 0 for cluster in stat: workers += len(cluster.workers) reincarnations += cluster.reincarnations # calculate tasks pm and avg exec time tasks_per = 0 per = _('day') exec_time = 0 last_tasks = models.Success.objects.filter(stopped__gte=timezone.now() - timedelta(hours=24)) tasks_per_day = last_tasks.count() if tasks_per_day > 0: # average execution time over the last 24 hours if not connection.vendor == 'sqlite': exec_time = last_tasks.aggregate(time_taken=Sum(F('stopped') - F('started'))) exec_time = exec_time['time_taken'].total_seconds() / tasks_per_day else: # can't sum timedeltas on sqlite for t in last_tasks: exec_time += t.time_taken() exec_time = exec_time / tasks_per_day # tasks per second/minute/hour/day in the last 24 hours if tasks_per_day > 24 * 60 * 60: tasks_per = tasks_per_day / (24 * 60 * 60) per = _('second') elif tasks_per_day > 24 * 60: tasks_per = tasks_per_day / (24 * 60) per = _('minute') elif tasks_per_day > 24: tasks_per = tasks_per_day / 24 per = _('hour') else: tasks_per = tasks_per_day # print to terminal print(term.clear_eos()) col_width = int(term.width / 6) print(term.black_on_green( term.center( _('-- {} {} on {} --').format(Conf.PREFIX.capitalize(), '.'.join(str(v) for v in VERSION), broker.info())))) print(term.cyan(_('Clusters')) + term.move_x(1 * col_width) + term.white(str(clusters)) + term.move_x(2 * col_width) + term.cyan(_('Workers')) + term.move_x(3 * col_width) + term.white(str(workers)) + term.move_x(4 * col_width) + term.cyan(_('Restarts')) + term.move_x(5 * col_width) + term.white(str(reincarnations)) ) print(term.cyan(_('Queued')) + term.move_x(1 * col_width) + term.white(str(broker.queue_size())) + term.move_x(2 * col_width) + term.cyan(_('Successes')) + term.move_x(3 * col_width) + term.white(str(models.Success.objects.count())) + term.move_x(4 * col_width) + term.cyan(_('Failures')) + term.move_x(5 * col_width) + term.white(str(models.Failure.objects.count())) ) print(term.cyan(_('Schedules')) + term.move_x(1 * col_width) + term.white(str(models.Schedule.objects.count())) + term.move_x(2 * col_width) + term.cyan(_('Tasks/{}'.format(per))) + term.move_x(3 * col_width) + term.white('{0:.2f}'.format(tasks_per)) + term.move_x(4 * col_width) + term.cyan(_('Avg time')) + term.move_x(5 * col_width) + term.white('{0:.4f}'.format(exec_time)) ) return True
def info(broker=None): if not broker: broker = get_broker() term = Terminal() broker.ping() stat = Stat.get_all(broker=broker) # general stats clusters = len(stat) workers = 0 reincarnations = 0 for cluster in stat: workers += len(cluster.workers) reincarnations += cluster.reincarnations # calculate tasks pm and avg exec time tasks_per = 0 per = _('day') exec_time = 0 last_tasks = models.Success.objects.filter(stopped__gte=timezone.now() - timedelta(hours=24)) tasks_per_day = last_tasks.count() if tasks_per_day > 0: # average execution time over the last 24 hours if not connection.vendor == 'sqlite': exec_time = last_tasks.aggregate( time_taken=Sum(F('stopped') - F('started'))) exec_time = exec_time['time_taken'].total_seconds() / tasks_per_day else: # can't sum timedeltas on sqlite for t in last_tasks: exec_time += t.time_taken() exec_time = exec_time / tasks_per_day # tasks per second/minute/hour/day in the last 24 hours if tasks_per_day > 24 * 60 * 60: tasks_per = tasks_per_day / (24 * 60 * 60) per = _('second') elif tasks_per_day > 24 * 60: tasks_per = tasks_per_day / (24 * 60) per = _('minute') elif tasks_per_day > 24: tasks_per = tasks_per_day / 24 per = _('hour') else: tasks_per = tasks_per_day # print to terminal print(term.clear_eos()) col_width = int(term.width / 6) print( term.black_on_green( term.center( _('-- {} {} on {} --').format( Conf.PREFIX.capitalize(), '.'.join(str(v) for v in VERSION), broker.info())))) print( term.cyan(_('Clusters')) + term.move_x(1 * col_width) + term.white(str(clusters)) + term.move_x(2 * col_width) + term.cyan(_('Workers')) + term.move_x(3 * col_width) + term.white(str(workers)) + term.move_x(4 * col_width) + term.cyan(_('Restarts')) + term.move_x(5 * col_width) + term.white(str(reincarnations))) print( term.cyan(_('Queued')) + term.move_x(1 * col_width) + term.white(str(broker.queue_size())) + term.move_x(2 * col_width) + term.cyan(_('Successes')) + term.move_x(3 * col_width) + term.white(str(models.Success.objects.count())) + term.move_x(4 * col_width) + term.cyan(_('Failures')) + term.move_x(5 * col_width) + term.white(str(models.Failure.objects.count()))) print( term.cyan(_('Schedules')) + term.move_x(1 * col_width) + term.white(str(models.Schedule.objects.count())) + term.move_x(2 * col_width) + term.cyan(_('Tasks/{}'.format(per))) + term.move_x(3 * col_width) + term.white('{0:.2f}'.format(tasks_per)) + term.move_x(4 * col_width) + term.cyan(_('Avg time')) + term.move_x(5 * col_width) + term.white('{0:.4f}'.format(exec_time))) return True
def debug(request): # 获取完整信息 full = request.GET.get('full') # 系统配置 sys_config = SysConfig().sys_config # MySQL信息 cursor = connection.cursor() mysql_info = { 'mysql_server_info': cursor.db.mysql_server_info, 'timezone_name': cursor.db.timezone_name } # Redis信息 redis_conn = get_redis_connection("default") full_redis_info = redis_conn.info() redis_info = { 'redis_version': full_redis_info.get('redis_version'), 'redis_mode': full_redis_info.get('redis_mode'), 'role': full_redis_info.get('role'), 'maxmemory_human': full_redis_info.get('maxmemory_human'), 'used_memory_human': full_redis_info.get('used_memory_human'), } # django_q django_q_version = '.'.join(str(i) for i in django_q.VERSION) broker = get_broker() stats = Stat.get_all(broker=broker) queue_size = broker.queue_size() lock_size = broker.lock_size() if lock_size: queue_size = '{}({})'.format(queue_size, lock_size) q_broker_stats = { 'info': broker.info(), 'Queued': queue_size, 'Success': Success.objects.count(), 'Failures': Failure.objects.count(), } q_cluster_stats = [] for stat in stats: # format uptime uptime = (timezone.now() - stat.tob).total_seconds() hours, remainder = divmod(uptime, 3600) minutes, seconds = divmod(remainder, 60) uptime = '%d:%02d:%02d' % (hours, minutes, seconds) q_cluster_stats.append({ 'host': stat.host, 'cluster_id': stat.cluster_id, 'state': stat.status, 'pool': len(stat.workers), 'tq': stat.task_q_size, 'rq': stat.done_q_size, 'rc': stat.reincarnations, 'up': uptime }) # Inception和goInception信息 inception_host = sys_config.get('inception_host') inception_port = sys_config.get('inception_port', 0) go_inception_host = sys_config.get('go_inception_host') go_inception_port = sys_config.get('go_inception_port', 0) inception_remote_backup_host = sys_config.get( 'inception_remote_backup_host', '') inception_remote_backup_port = sys_config.get( 'inception_remote_backup_port', '') inception_remote_backup_user = sys_config.get( 'inception_remote_backup_user', '') inception_remote_backup_password = sys_config.get( 'inception_remote_backup_password', '') # inception try: inc_conn = MySQLdb.connect(host=inception_host, port=int(inception_port), connect_timeout=1, cursorclass=MySQLdb.cursors.DictCursor) cursor = inc_conn.cursor() cursor.execute('inception get variables') rows = cursor.fetchall() full_inception_info = dict() for row in rows: full_inception_info[row.get('Variable_name')] = row.get('Value') inception_info = { 'version': full_inception_info.get('version'), 'max_allowed_packet': full_inception_info.get('max_allowed_packet'), 'inception_language_code': full_inception_info.get('inception_language_code'), 'inception_osc_on': full_inception_info.get('inception_osc_on'), 'inception_osc_bin_dir': full_inception_info.get('inception_osc_bin_dir'), } except Exception as e: inception_info = f'获取Inception信息报错:{e}' full_inception_info = inception_info # goInception try: goinc_conn = MySQLdb.connect(host=go_inception_host, port=int(go_inception_port), connect_timeout=1, cursorclass=MySQLdb.cursors.DictCursor) cursor = goinc_conn.cursor() cursor.execute('inception get variables') rows = cursor.fetchall() full_goinception_info = dict() for row in rows: full_goinception_info[row.get('Variable_name')] = row.get('Value') goinception_info = { 'version': full_goinception_info.get('version'), 'max_allowed_packet': full_goinception_info.get('max_allowed_packet'), 'lang': full_goinception_info.get('lang'), 'osc_on': full_goinception_info.get('osc_on'), 'osc_bin_dir': full_goinception_info.get('osc_bin_dir'), 'ghost_on': full_goinception_info.get('ghost_on'), } except Exception as e: goinception_info = f'获取goInception信息报错:{e}' full_goinception_info = goinception_info # 备份库 try: bak_conn = MySQLdb.connect(host=inception_remote_backup_host, port=int(inception_remote_backup_port), user=inception_remote_backup_user, password=inception_remote_backup_password, connect_timeout=1) cursor = bak_conn.cursor() cursor.execute('select 1;') backup_info = 'normal' except Exception as e: backup_info = f'无法连接Inception备份库\n{e}' # PACKAGES installed_packages = pkg_resources.working_set installed_packages_list = sorted( ["%s==%s" % (i.key, i.version) for i in installed_packages]) # 最终集合 system_info = { 'archery': { 'version': archery.display_version }, 'django_q': { 'version': django_q_version, 'conf': django_q.conf.Conf.conf, 'q_cluster_stats': q_cluster_stats if q_cluster_stats else '没有正在运行的集群信息,请检查django_q状态', 'q_broker_stats': q_broker_stats }, 'inception': { 'enable_inception': sys_config.get('inception'), 'inception_info': full_inception_info if full else inception_info, 'goinception_info': full_goinception_info if full else goinception_info, 'backup_info': backup_info }, 'runtime_info': { 'python_version': platform.python_version(), 'mysql_info': mysql_info, 'redis_info': full_redis_info if full else redis_info, 'sys_argv': sys.argv, 'platform': platform.uname() }, 'sys_config': sys_config, 'packages': installed_packages_list } return JsonResponse(system_info)
def stat(self): if self.sentinel: return Stat.get(pid=self.pid, cluster_id=self.cluster_id) return Status(pid=self.pid, cluster_id=self.cluster_id)