Пример #1
0
def get_current_task():
    clusters = Stat.get_all()
    current_tasks = CurrentTask.objects.all()

    if not clusters or clusters[0].status == 'Stopped':
        [current_task.delete() for current_task in current_tasks]
        current_tasks = []

    now = datetime.datetime.now(tz=datetime.timezone.utc)
    tasks = []
    for task in current_tasks:
        if task.ended:
            task.delete()
        else:
            running_for = (now - task.started).seconds
            progress = '{}/{}'.format(task.progress_current, task.progress_max)
            tasks.append({
                'action': task.name,
                'running_for': running_for,
                'progress': progress
            })
            if task.progress_current == task.progress_max:
                task.ended = True
                task.save()

    return tasks
Пример #2
0
def is_worker_running(**kwargs):
    """
    Return True if the background worker process is oprational
    """

    clusters = Stat.get_all()

    if len(clusters) > 0:
        # TODO - Introspect on any cluster information
        return True

    """
    Sometimes Stat.get_all() returns [].
    In this case we have the 'heartbeat' task running every 15 minutes.
    Check to see if we have a result within the last 20 minutes
    """

    now = datetime.now()
    past = now - timedelta(minutes=20)

    results = Success.objects.filter(
        func='InvenTree.tasks.heartbeat',
        started__gte=past
    )

    # If any results are returned, then the background worker is running!
    return results.exists()
Пример #3
0
def test_bad_secret(r, monkeypatch):
    list_key = 'test_bad_secret'
    async('math.copysign', 1, -1, list_key=list_key)
    stop_event = Event()
    stop_event.set()
    start_event = Event()
    s = Sentinel(stop_event, start_event, list_key=list_key, start=False)
    Stat(s).save()
    # change the SECRET
    monkeypatch.setattr(Conf, "SECRET_KEY", "OOPS")
    stat = Stat.get_all(r)
    assert len(stat) == 0
    assert Stat.get(s.parent_pid, r) is None
    task_queue = Queue()
    pusher(task_queue, stop_event, list_key=list_key, r=r)
    result_queue = Queue()
    task_queue.put('STOP')
    worker(task_queue, result_queue, Value('f', -1), )
    assert result_queue.qsize() == 0
    r.delete(list_key)
Пример #4
0
 def spawn_cluster(self):
     self.pool = []
     Stat(self).save()
     # spawn worker pool
     for i in range(self.pool_size):
         self.spawn_worker()
     # spawn auxiliary
     self.monitor = self.spawn_monitor()
     self.pusher = self.spawn_pusher()
     # set worker cpu affinity if needed
     if psutil and Conf.CPU_AFFINITY:
         set_cpu_affinity(Conf.CPU_AFFINITY, [w.pid for w in self.pool])
Пример #5
0
 def stop(self):
     Stat(self).save()
     name = current_process().name
     logger.info('{} stopping cluster processes'.format(name))
     # Stopping pusher
     self.event_out.set()
     # Wait for it to stop
     while self.pusher.is_alive():
         sleep(0.2)
         Stat(self).save()
     # Put poison pills in the queue
     for _ in range(len(self.pool)):
         self.task_queue.put('STOP')
     self.task_queue.close()
     # wait for the task queue to empty
     self.task_queue.join_thread()
     # Wait for all the workers to exit
     while len(self.pool):
         for p in self.pool:
             if not p.is_alive():
                 self.pool.remove(p)
         sleep(0.2)
         Stat(self).save()
     # Finally stop the monitor
     self.result_queue.put('STOP')
     self.result_queue.close()
     # Wait for the result queue to empty
     self.result_queue.join_thread()
     logger.info('{} waiting for the monitor.'.format(name))
     # Wait for everything to close or time out
     count = 0
     if not self.timeout:
         self.timeout = 30
     while self.status() == Conf.STOPPING and count < self.timeout * 5:
         sleep(0.2)
         Stat(self).save()
         count += 1
     # Final status
     Stat(self).save()
Пример #6
0
def test_bad_secret(r, monkeypatch):
    list_key = 'test_bad_secret'
    async ('math.copysign', 1, -1, list_key=list_key)
    stop_event = Event()
    stop_event.set()
    start_event = Event()
    s = Sentinel(stop_event, start_event, list_key=list_key, start=False)
    Stat(s).save()
    # change the SECRET
    monkeypatch.setattr(Conf, "SECRET_KEY", "OOPS")
    stat = Stat.get_all(r)
    assert len(stat) == 0
    assert Stat.get(s.parent_pid, r) is None
    task_queue = Queue()
    pusher(task_queue, stop_event, list_key=list_key, r=r)
    result_queue = Queue()
    task_queue.put('STOP')
    worker(
        task_queue,
        result_queue,
        Value('f', -1),
    )
    assert result_queue.qsize() == 0
    r.delete(list_key)
Пример #7
0
def test_monitor():
    assert Stat.get(0).sentinel == 0
    c = Cluster()
    c.start()
    stats = monitor(run_once=True)
    c.stop()
    assert len(stats) > 0
    found_c = False
    for stat in stats:
        if stat.cluster_id == c.pid:
            found_c = True
            assert stat.uptime() > 0
            assert stat.empty_queues() is True
            break
    assert found_c is True
Пример #8
0
 def guard(self):
     logger.info(
         _('{} guarding cluster at {}').format(current_process().name,
                                               self.pid))
     self.start_event.set()
     Stat(self).save()
     logger.info(_('Q Cluster-{} running.').format(self.parent_pid))
     scheduler(list_key=self.list_key)
     counter = 0
     # Guard loop. Runs at least once
     while not self.stop_event.is_set() or not counter:
         # Check Workers
         for p in self.pool:
             # Are you alive?
             if not p.is_alive() or (self.timeout
                                     and int(p.timer.value) == 0):
                 self.reincarnate(p)
                 continue
             # Decrement timer if work is being done
             if p.timer.value > 0:
                 p.timer.value -= 1
         # Check Monitor
         if not self.monitor.is_alive():
             self.reincarnate(self.monitor)
         # Check Pusher
         if not self.pusher.is_alive():
             self.reincarnate(self.pusher)
         # Call scheduler once a minute (or so)
         counter += 1
         if counter > 120:
             counter = 0
             scheduler(list_key=self.list_key)
         # Save current status
         Stat(self).save()
         sleep(0.5)
     self.stop()
Пример #9
0
def test_monitor():
    assert Stat.get(0).sentinel == 0
    c = Cluster()
    c.start()
    stats = monitor(run_once=True)
    c.stop()
    assert len(stats) > 0
    found_c = False
    for stat in stats:
        if stat.cluster_id == c.pid:
            found_c = True
            assert stat.uptime() > 0
            assert stat.empty_queues() is True
            break
    assert found_c is True
Пример #10
0
 def stat(self):
     if self.sentinel:
         return Stat.get(self.pid)
     return Status(self.pid)
Пример #11
0
 def stat(self):
     if self.sentinel:
         return Stat.get(self.pid)
     return Status(self.pid)