def monitor_system(): """ This task used by scheduler to monitor state of the system. :return: """ last_action = models.ManagementAction.objects.filter(ping_index__isnull=False).last() if last_action: ping_index = last_action.ping_index + 1 else: ping_index = 0 # TODO: Handle the case where host manager has not responded to last and itself has died _ = app.send_task('manage_host', args=['list', ping_index], exchange='qmanager') worker_stats = {'alive':0, 'transition':0, 'dead': models.Worker.objects.filter(alive=False).count() } for w in models.Worker.objects.filter(alive=True): # if worker is not heard from via manager for more than 10 minutes # mark it as dead, so that processes_monitor can mark tasks are errored and restart if possible. if (timezone.now() - w.last_ping).total_seconds() > 600: w.alive = False w.save() worker_stats['transition'] += 1 else: worker_stats['alive'] += 1 process_stats = {'processes': models.DVAPQL.objects.count(), 'completed_processes': models.DVAPQL.objects.filter(completed=True).count(), 'tasks': models.TEvent.objects.count(), 'pending_tasks': models.TEvent.objects.filter(started=False).count(), 'completed_tasks': models.TEvent.objects.filter(started=True, completed=True).count()} _ = models.SystemState.objects.create(redis_stats=redis_client.info(), process_stats=process_stats, worker_stats=worker_stats)
def monitor_system(): """ This task used by scheduler to monitor state of the system. :return: """ last_action = models.ManagementAction.objects.filter(ping_index__isnull=False).last() if last_action: ping_index = last_action.ping_index + 1 else: ping_index = 0 # TODO: Handle the case where host manager has not responded to last and itself has died _ = app.send_task('manage_host', args=['list', ping_index], exchange='qmanager') process_stats = {'processes': models.DVAPQL.objects.count(), 'completed_processes': models.DVAPQL.objects.filter(completed=True).count(), 'tasks': models.TEvent.objects.count(), 'pending_tasks': models.TEvent.objects.filter(started=False).count(), 'completed_tasks': models.TEvent.objects.filter(started=True, completed=True).count()} _ = models.SystemState.objects.create(redis_stats=redis_client.info(),process_stats=process_stats)