def guard(self): logger.info(_('{} guarding cluster at {}').format(current_process().name, self.pid)) self.start_event.set() Stat(self).save() logger.info(_('Q Cluster-{} running.').format(self.parent_pid)) scheduler(broker=self.broker) counter = 0 cycle = Conf.GUARD_CYCLE # guard loop sleep in seconds # Guard loop. Runs at least once while not self.stop_event.is_set() or not counter: # Check Workers for p in self.pool: # Are you alive? if not p.is_alive() or (self.timeout and p.timer.value == 0): self.reincarnate(p) continue # Decrement timer if work is being done if self.timeout and p.timer.value > 0: p.timer.value -= cycle # Check Monitor if not self.monitor.is_alive(): self.reincarnate(self.monitor) # Check Pusher if not self.pusher.is_alive(): self.reincarnate(self.pusher) # Call scheduler once a minute (or so) counter += cycle if counter >= 30 and Conf.SCHEDULER: counter = 0 scheduler(broker=self.broker) # Save current status Stat(self).save() sleep(cycle) self.stop()
def guard(self): logger.info( _(f"{current_process().name} guarding cluster {humanize(self.cluster_id.hex)}" )) self.start_event.set() Stat(self).save() logger.info(_(f"Q Cluster {humanize(self.cluster_id.hex)} running.")) counter = 0 cycle = Conf.GUARD_CYCLE # guard loop sleep in seconds # Guard loop. Runs at least once while not self.stop_event.is_set() or not counter: # Check Workers for p in self.pool: with p.timer.get_lock(): # Check if worker is alive or timed out if not p.is_alive() or p.timer.value == 0: self.reincarnate_worker(p) continue # Decrement timer if work is being done if p.timer.value > 0: p.timer.value = 0 if p.timer.value < cycle else p.timer.value - cycle # Check Monitor if not self.monitor.is_alive(): self.reincarnate_monitor() # Check Pusher if not self.pusher.is_alive(): self.reincarnate_pusher() counter += cycle # Run every 30 seconds if counter >= 30: # Update cluster heartbeat time now_time = timezone.now() self.cluster_model.heartbeat_time = now_time self.cluster_model.save(update_fields=['heartbeat_time']) # Clean up other cluster instances that have died self.cluster_model.filter(heartbeat_time__lt=now_time - timedelta(minutes=1)).delete() # Call scheduler if Conf.SCHEDULER: scheduler(broker=self.broker) counter = 0 # Save current status Stat(self).save() sleep(cycle) self.stop()
def test_bad_secret(broker, monkeypatch): broker.list_key = "test_bad_secret:q" async_task("math.copysign", 1, -1, broker=broker) stop_event = Event() stop_event.set() start_event = Event() cluster_id = uuidlib.uuid4() s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker, start=False) Stat(s).save() # change the SECRET monkeypatch.setattr(Conf, "SECRET_KEY", "OOPS") stat = Stat.get_all() assert len(stat) == 0 assert Stat.get(pid=s.parent_pid, cluster_id=cluster_id) is None task_queue = Queue() pusher(task_queue, stop_event, broker=broker) result_queue = Queue() task_queue.put("STOP") worker( task_queue, result_queue, Value("f", -1), ) assert result_queue.qsize() == 0 broker.delete_queue()
def test_bad_broker(broker, mocker): mocker.patch.object(broker, 'set_stat', side_effect=Exception('Unusable connection')) stop_event = Event() stop_event.set() start_event = Event() s = Sentinel(stop_event, start_event, broker=broker, start=False) mock_close = mocker.patch.object(broker, 'close') Stat(s).save() assert mock_close.called
def stop(self): Stat(self).save() name = current_process().name logger.info(_('{} stopping cluster processes').format(name)) # Stopping pusher self.event_out.set() # Wait for it to stop while self.pusher.is_alive(): sleep(0.1) Stat(self).save() # Put poison pills in the queue for __ in range(len(self.pool)): self.task_queue.put('STOP') self.task_queue.close() # wait for the task queue to empty self.task_queue.join_thread() # Wait for all the workers to exit while len(self.pool): for p in self.pool: if not p.is_alive(): self.pool.remove(p) sleep(0.1) Stat(self).save() # Finally stop the monitor self.result_queue.put('STOP') self.result_queue.close() # Wait for the result queue to empty self.result_queue.join_thread() logger.info(_('{} waiting for the monitor.').format(name)) # Wait for everything to close or time out count = 0 if not self.timeout: self.timeout = 30 while self.status() == Conf.STOPPING and count < self.timeout * 10: sleep(0.1) Stat(self).save() count += 1 # Final status Stat(self).save()
def spawn_cluster(self): self.pool = [] Stat(self).save() db.connection.close() # spawn worker pool for __ in range(self.pool_size): self.spawn_worker() # spawn auxiliary self.monitor = self.spawn_monitor() self.pusher = self.spawn_pusher() # set worker cpu affinity if needed if psutil and Conf.CPU_AFFINITY: set_cpu_affinity(Conf.CPU_AFFINITY, [w.pid for w in self.pool])
def guard(self): logger.info( _( f"{current_process().name} guarding cluster {humanize(self.cluster_id.hex)}" ) ) self.start_event.set() Stat(self).save() logger.info(_(f"Q Cluster {humanize(self.cluster_id.hex)} running.")) counter = 0 cycle = Conf.GUARD_CYCLE # guard loop sleep in seconds # Guard loop. Runs at least once while not self.stop_event.is_set() or not counter: # Check Workers for p in self.pool: with p.timer.get_lock(): # Are you alive? if not p.is_alive() or p.timer.value == 0: self.reincarnate(p) continue # Decrement timer if work is being done if p.timer.value > 0: p.timer.value -= cycle # Check Monitor if not self.monitor.is_alive(): self.reincarnate(self.monitor) # Check Pusher if not self.pusher.is_alive(): self.reincarnate(self.pusher) # Call scheduler once a minute (or so) counter += cycle if counter >= 30 and Conf.SCHEDULER: counter = 0 scheduler(broker=self.broker) # Save current status Stat(self).save() sleep(cycle) self.stop()
def test_bad_secret(broker, monkeypatch): broker.list_key = 'test_bad_secret:q' async('math.copysign', 1, -1, broker=broker) stop_event = Event() stop_event.set() start_event = Event() s = Sentinel(stop_event, start_event, broker=broker, start=False) Stat(s).save() # change the SECRET monkeypatch.setattr(Conf, "SECRET_KEY", "OOPS") stat = Stat.get_all() assert len(stat) == 0 assert Stat.get(s.parent_pid) is None task_queue = Queue() pusher(task_queue, stop_event, broker=broker) result_queue = Queue() task_queue.put('STOP') worker(task_queue, result_queue, Value('f', -1), ) assert result_queue.qsize() == 0 broker.delete_queue()