def get_worker_count(request=None):
"""
Return a count of the active Celery workers.
"""
# Inner imports so we don't risk circular imports
from nautobot.core.celery import app # noqa
from rq.worker import Worker # noqa
from django_rq.queues import get_connection # noqa
# Try RQ first since, it's faster.
rq_count = Worker.count(get_connection("default"))
# Celery next, since it's slower.
inspect = app.control.inspect()
active = inspect.active() # None if no active workers
celery_count = len(active) if active is not None else 0
if rq_count and not celery_count:
if request:
messages.warning(
request,
"RQ workers are deprecated. Please migrate your workers to Celery."
)
return celery_count
def workers_status(request):
inspect = app.control.inspect()
latestpycm = models.PyCompmusicVersion.objects.order_by('-commit_date').first()
latestessentia = models.EssentiaVersion.objects.order_by('-commit_date').first()
try:
hosts = inspect.active()
except:
hosts = None
workerobs = models.Worker.objects.all()
workerkeys = ["celery@%s" % w.hostname for w in workerobs]
if hosts:
hostkeys = hosts.keys()
workers = list(set(workerkeys) & set(hostkeys))
neww = []
for w in workers:
host = w.split("@")[1]
theworker = workerobs.get(hostname=host)
essentia = theworker.essentia
pyc = theworker.pycompmusic
num_proc = len(hosts[w])
if theworker.state == models.Worker.UPDATING:
state = "Updating"
elif num_proc:
state = "Active"
else:
state = "Idle"
if essentia:
e = {'version': essentia.sha1, 'link': essentia.short_link()}
else:
e = {}
if pyc:
p = {'version': pyc.sha1, 'link': pyc.short_link()}
else:
p = {}
neww.append({"host": host,
"number": num_proc,
"state": state,
"worker": {'essentia': e, 'pyc': p}})
workers = neww
newworkers = list(set(hostkeys) - set(workerkeys))
newworkers = [w.split("@")[1] for w in newworkers]
inactiveworkers = list(set(workerkeys) - set(hostkeys))
inactiveworkers = [w.split("@")[1] for w in inactiveworkers]
else:
workers = []
newworkers = []
inactiveworkers = [w.split("@")[1] for w in workerkeys]
ret = {"workers": workers, "newworkers": newworkers, "inactiveworkers": inactiveworkers,
"latestpycm": latestpycm.sha1, "latestessentia": latestessentia.sha1}
return HttpResponse(json.dumps(ret), content_type='application/json')
def workers_status(request):
inspect = app.control.inspect()
latestpycm = models.PyCompmusicVersion.objects.order_by('-commit_date').first()
latestessentia = models.EssentiaVersion.objects.order_by('-commit_date').first()
try:
hosts = inspect.active()
except:
hosts = None
workerobs = models.Worker.objects.all()
workerkeys = ["celery@%s" % w.hostname for w in workerobs]
if hosts:
hostkeys = hosts.keys()
workers = list(set(workerkeys) & set(hostkeys))
neww = []
for w in workers:
host = w.split("@")[1]
theworker = workerobs.get(hostname=host)
essentia = theworker.essentia
pyc = theworker.pycompmusic
num_proc = len(hosts[w])
if theworker.state == models.Worker.UPDATING:
state = "Updating"
elif num_proc:
state = "Active"
else:
state = "Idle"
if essentia:
e = {'version': essentia.sha1, 'link': essentia.short_link()}
else:
e = {}
if pyc:
p = {'version': pyc.sha1, 'link': pyc.short_link()}
else:
p = {}
neww.append({"host": host,
"number": num_proc,
"state": state,
"worker": {'essentia': e, 'pyc': p}})
workers = neww
newworkers = list(set(hostkeys) - set(workerkeys))
newworkers = [w.split("@")[1] for w in newworkers]
inactiveworkers = list(set(workerkeys) - set(hostkeys))
inactiveworkers = [w.split("@")[1] for w in inactiveworkers]
else:
workers = []
newworkers = []
inactiveworkers = [w.split("@")[1] for w in workerkeys]
ret = {"workers": workers, "newworkers": newworkers, "inactiveworkers": inactiveworkers,
"latestpycm": latestpycm.sha1, "latestessentia": latestessentia.sha1}
return HttpResponse(json.dumps(ret), content_type='application/json')
def manager(request):
# Add a new worker to the cluster
register = request.GET.get("register")
if register is not None:
jobs.register_host.apply_async([register], queue=register)
return redirect('docserver-manager')
if request.method == "POST":
# Update essentia and pycompmusic on all workers
update = request.POST.get("updateall")
if update is not None:
jobs.update_all_workers(request.user.username)
return redirect('docserver-manager')
# Process a module version
run = request.POST.get("run")
if run is not None:
jobs.run_module(int(run))
return redirect('docserver-manager')
modules = models.Module.objects.all().order_by('name')
collections = models.Collection.objects.all()
inspect = app.control.inspect()
# TODO: Load the task data via ajax, so the page loads quickly
try:
hosts = inspect.active()
except:
hosts = None
workerobs = models.Worker.objects.all()
workerkeys = ["celery@%s" % w.hostname for w in workerobs]
if hosts:
hostkeys = hosts.keys()
workers = list(set(workerkeys) & set(hostkeys))
neww = []
for w in workers:
host = w.split("@")[1]
theworker = workerobs.get(hostname=host)
num_proc = len(hosts[w])
if theworker.state == models.Worker.UPDATING:
state = "Updating"
elif num_proc:
state = "Active"
else:
state = "Idle"
neww.append({"host": host,
"number": num_proc,
"state": state,
"worker": theworker})
workers = neww
newworkers = list(set(hostkeys) - set(workerkeys))
newworkers = [w.split("@")[1] for w in newworkers]
inactiveworkers = list(set(workerkeys) - set(hostkeys))
inactiveworkers = [w.split("@")[1] for w in inactiveworkers]
else:
workers = []
newworkers = []
inactiveworkers = [w.split("@")[1] for w in workerkeys]
latestpycm = models.PyCompmusicVersion.objects.order_by('-commit_date').first()
latestessentia = models.EssentiaVersion.objects.order_by('-commit_date').first()
ret = {"modules": modules, "collections": collections, "workers": workers,
"newworkers": newworkers, "inactiveworkers": inactiveworkers,
"latestpycm": latestpycm, "latestessentia": latestessentia}
return render(request, 'docserver/manager.html', ret)
def monitorWorkers( manager ):
print( "Monitorig: " + manager )
while True:
numWorkers = 0
numTasks = 0
utilization = 0
busyWorkers = []
allWorkers = []
app = Celery('airfoilApp', backend='rpc://', broker=manager)
inspect = app.control.inspect()
try:
workers = inspect.ping()
except:
print("\tReconnecting...")
app = Celery('airfoilApp', backend='rpc://', broker=manager)
inspect = app.control.inspect()
workers = inspect.ping()
if not workers is None:
numWorkers = len( workers )
for worker in workers:
allWorkers.append( getWorkerName(worker) )
print( worker + " - Online" )
# Count active tasks.
allTasks = inspect.active()
if not allTasks is None:
for workerId in allTasks:
workerTasks = allTasks[workerId]
numTasks = numTasks + len( workerTasks )
if len( workerTasks ) != 0:
busyWorkers.append( getWorkerName(workerId) )
# Count queued tasks too.
allTasks = inspect.reserved()
if not allTasks is None:
for workerId in allTasks:
workerTasks = allTasks[workerId]
numTasks = numTasks + len( workerTasks )
# for task in workerTasks:
# print( type(task) )
# print( task['id'], task['name'] )
if numWorkers != 0:
utilization = numTasks / numWorkers
panic = (utilization > 1)
print( "Workers :", numWorkers )
print( "Tasks :", numTasks )
if panic:
print( "Utilization : %3.1f %% Panic!!! " % (utilization * 100.0) )
else:
print( "Utilization : %3.1f %%" % (utilization * 100.0) )
if( utilization >= WORKERS_UPPER_LIMIT and numWorkers < WORKERS_MAX ):
addMoreWorkers( allWorkers, panic )
else:
if( utilization <= WORKERS_LOWER_LIMIT and numWorkers > WORKERS_MIN ):
releaseWorkers( allWorkers, busyWorkers )
print( "===========================================" )
time.sleep(5)
