def monitor_system():
"""
This task used by scheduler to monitor state of the system.
:return:
"""
last_action = models.ManagementAction.objects.filter(ping_index__isnull=False).last()
if last_action:
ping_index = last_action.ping_index + 1
else:
ping_index = 0
# TODO: Handle the case where host manager has not responded to last and itself has died
_ = app.send_task('manage_host', args=['list', ping_index], exchange='qmanager')
worker_stats = {'alive':0,
'transition':0,
'dead': models.Worker.objects.filter(alive=False).count()
}
for w in models.Worker.objects.filter(alive=True):
# if worker is not heard from via manager for more than 10 minutes
# mark it as dead, so that processes_monitor can mark tasks are errored and restart if possible.
if (timezone.now() - w.last_ping).total_seconds() > 600:
w.alive = False
w.save()
worker_stats['transition'] += 1
else:
worker_stats['alive'] += 1
process_stats = {'processes': models.DVAPQL.objects.count(),
'completed_processes': models.DVAPQL.objects.filter(completed=True).count(),
'tasks': models.TEvent.objects.count(),
'pending_tasks': models.TEvent.objects.filter(started=False).count(),
'completed_tasks': models.TEvent.objects.filter(started=True, completed=True).count()}
_ = models.SystemState.objects.create(redis_stats=redis_client.info(),
process_stats=process_stats,
worker_stats=worker_stats)
def monitor_system():
"""
This task used by scheduler to monitor state of the system.
:return:
"""
last_action = models.ManagementAction.objects.filter(ping_index__isnull=False).last()
if last_action:
ping_index = last_action.ping_index + 1
else:
ping_index = 0
# TODO: Handle the case where host manager has not responded to last and itself has died
_ = app.send_task('manage_host', args=['list', ping_index], exchange='qmanager')
process_stats = {'processes': models.DVAPQL.objects.count(),
'completed_processes': models.DVAPQL.objects.filter(completed=True).count(),
'tasks': models.TEvent.objects.count(),
'pending_tasks': models.TEvent.objects.filter(started=False).count(),
'completed_tasks': models.TEvent.objects.filter(started=True, completed=True).count()}
_ = models.SystemState.objects.create(redis_stats=redis_client.info(),process_stats=process_stats)
