def get_current_task():
clusters = Stat.get_all()
current_tasks = CurrentTask.objects.all()
if not clusters or clusters[0].status == 'Stopped':
[current_task.delete() for current_task in current_tasks]
current_tasks = []
now = datetime.datetime.now(tz=datetime.timezone.utc)
tasks = []
for task in current_tasks:
if task.ended:
task.delete()
else:
running_for = (now - task.started).seconds
progress = '{}/{}'.format(task.progress_current, task.progress_max)
tasks.append({
'action': task.name,
'running_for': running_for,
'progress': progress
})
if task.progress_current == task.progress_max:
task.ended = True
task.save()
return tasks
def is_worker_running(**kwargs):
"""
Return True if the background worker process is oprational
"""
clusters = Stat.get_all()
if len(clusters) > 0:
# TODO - Introspect on any cluster information
return True
"""
Sometimes Stat.get_all() returns [].
In this case we have the 'heartbeat' task running every 15 minutes.
Check to see if we have a result within the last 20 minutes
"""
now = datetime.now()
past = now - timedelta(minutes=20)
results = Success.objects.filter(
func='InvenTree.tasks.heartbeat',
started__gte=past
)
# If any results are returned, then the background worker is running!
return results.exists()
def test_bad_secret(r, monkeypatch):
list_key = 'test_bad_secret'
async('math.copysign', 1, -1, list_key=list_key)
stop_event = Event()
stop_event.set()
start_event = Event()
s = Sentinel(stop_event, start_event, list_key=list_key, start=False)
Stat(s).save()
# change the SECRET
monkeypatch.setattr(Conf, "SECRET_KEY", "OOPS")
stat = Stat.get_all(r)
assert len(stat) == 0
assert Stat.get(s.parent_pid, r) is None
task_queue = Queue()
pusher(task_queue, stop_event, list_key=list_key, r=r)
result_queue = Queue()
task_queue.put('STOP')
worker(task_queue, result_queue, Value('f', -1), )
assert result_queue.qsize() == 0
r.delete(list_key)
def spawn_cluster(self):
self.pool = []
Stat(self).save()
# spawn worker pool
for i in range(self.pool_size):
self.spawn_worker()
# spawn auxiliary
self.monitor = self.spawn_monitor()
self.pusher = self.spawn_pusher()
# set worker cpu affinity if needed
if psutil and Conf.CPU_AFFINITY:
set_cpu_affinity(Conf.CPU_AFFINITY, [w.pid for w in self.pool])
def stop(self):
Stat(self).save()
name = current_process().name
logger.info('{} stopping cluster processes'.format(name))
# Stopping pusher
self.event_out.set()
# Wait for it to stop
while self.pusher.is_alive():
sleep(0.2)
Stat(self).save()
# Put poison pills in the queue
for _ in range(len(self.pool)):
self.task_queue.put('STOP')
self.task_queue.close()
# wait for the task queue to empty
self.task_queue.join_thread()
# Wait for all the workers to exit
while len(self.pool):
for p in self.pool:
if not p.is_alive():
self.pool.remove(p)
sleep(0.2)
Stat(self).save()
# Finally stop the monitor
self.result_queue.put('STOP')
self.result_queue.close()
# Wait for the result queue to empty
self.result_queue.join_thread()
logger.info('{} waiting for the monitor.'.format(name))
# Wait for everything to close or time out
count = 0
if not self.timeout:
self.timeout = 30
while self.status() == Conf.STOPPING and count < self.timeout * 5:
sleep(0.2)
Stat(self).save()
count += 1
# Final status
Stat(self).save()
def test_bad_secret(r, monkeypatch):
list_key = 'test_bad_secret'
async ('math.copysign', 1, -1, list_key=list_key)
stop_event = Event()
stop_event.set()
start_event = Event()
s = Sentinel(stop_event, start_event, list_key=list_key, start=False)
Stat(s).save()
# change the SECRET
monkeypatch.setattr(Conf, "SECRET_KEY", "OOPS")
stat = Stat.get_all(r)
assert len(stat) == 0
assert Stat.get(s.parent_pid, r) is None
task_queue = Queue()
pusher(task_queue, stop_event, list_key=list_key, r=r)
result_queue = Queue()
task_queue.put('STOP')
worker(
task_queue,
result_queue,
Value('f', -1),
)
assert result_queue.qsize() == 0
r.delete(list_key)
def test_monitor():
assert Stat.get(0).sentinel == 0
c = Cluster()
c.start()
stats = monitor(run_once=True)
c.stop()
assert len(stats) > 0
found_c = False
for stat in stats:
if stat.cluster_id == c.pid:
found_c = True
assert stat.uptime() > 0
assert stat.empty_queues() is True
break
assert found_c is True
def guard(self):
logger.info(
_('{} guarding cluster at {}').format(current_process().name,
self.pid))
self.start_event.set()
Stat(self).save()
logger.info(_('Q Cluster-{} running.').format(self.parent_pid))
scheduler(list_key=self.list_key)
counter = 0
# Guard loop. Runs at least once
while not self.stop_event.is_set() or not counter:
# Check Workers
for p in self.pool:
# Are you alive?
if not p.is_alive() or (self.timeout
and int(p.timer.value) == 0):
self.reincarnate(p)
continue
# Decrement timer if work is being done
if p.timer.value > 0:
p.timer.value -= 1
# Check Monitor
if not self.monitor.is_alive():
self.reincarnate(self.monitor)
# Check Pusher
if not self.pusher.is_alive():
self.reincarnate(self.pusher)
# Call scheduler once a minute (or so)
counter += 1
if counter > 120:
counter = 0
scheduler(list_key=self.list_key)
# Save current status
Stat(self).save()
sleep(0.5)
self.stop()
def test_monitor():
assert Stat.get(0).sentinel == 0
c = Cluster()
c.start()
stats = monitor(run_once=True)
c.stop()
assert len(stats) > 0
found_c = False
for stat in stats:
if stat.cluster_id == c.pid:
found_c = True
assert stat.uptime() > 0
assert stat.empty_queues() is True
break
assert found_c is True
def stat(self):
if self.sentinel:
return Stat.get(self.pid)
return Status(self.pid)
def stat(self):
if self.sentinel:
return Stat.get(self.pid)
return Status(self.pid)
