def run(self): if not self.config: return while not self._stop: for buffer_name, buffer_config in self.config.items(): # 需要停止 if self._stop: return self.buffer_queue = ztq_core.get_buffer_queue(buffer_name) self.task_queue = ztq_core.get_task_queue(buffer_name) self.buffer_name = buffer_name self.task_queue_limit = int(buffer_config["thread_limit"]) while True: try: self.start_job() break except ztq_core.ConnectionError: time.sleep(3) time.sleep(1)
def get_taskqueues_list(): # 队列情况列表 dispatcher_config = ztq_core.get_dispatcher_config() queue_weight = dispatcher_config['queue_weight'] queues_list = ztq_core.get_queue_config() # 排序 sort_queue_name = {} for queue_name, queue_config in queues_list.items(): sort_queue_name[queue_name] = len(ztq_core.get_error_queue(queue_name)) for queue_name in sorted(sort_queue_name, key=lambda x: sort_queue_name[x], reverse=True): task_queue = {} task_queue['name'] = queue_name #task_queue['tags'] = queue_config.get('tags',()) queue = ztq_core.get_task_queue(queue_name) # 任务数/错误数 task_queue['length'] = len(queue) task_queue['error_length'] = sort_queue_name[queue_name] #任务首个时间 task_queue['error_end'] = task_queue['first'] = '' first_job = queue[0] first_job = ztq_core.get_task_hash(queue_name).get(first_job) if first_job: task_queue['first'] = datetime.datetime.fromtimestamp( first_job['runtime'].get('create', 0)) #错误最末一个的时间 error_first_job = ztq_core.get_error_queue(queue_name)[0] error_first_job = ztq_core.get_error_hash(queue_name).get( error_first_job) if error_first_job: task_queue['error_end'] = datetime.datetime.fromtimestamp( error_first_job['runtime'].get('create', 0)) task_queue['weight'] = queue_weight.get(queue_name, 0) # 获取worker工作线程配置 workers_config = ztq_core.get_worker_config() task_queue['from_right'] = True for worker_name, worker_config in workers_config.items(): task_queue['workers'] = [] for config in worker_config.get(queue_name, []): task_queue['workers'].append( [worker_name + ':', config['interval']]) if 'from_right' in config: task_queue['from_right'] = config['from_right'] task_queue['buffer_length'] = len( ztq_core.get_buffer_queue(queue_name)) yield task_queue
def get_taskqueues_list(): # 队列情况列表 dispatcher_config = ztq_core.get_dispatcher_config() queue_weight = dispatcher_config['queue_weight'] queues_list = ztq_core.get_queue_config() # 排序 sort_queue_name = {} for queue_name, queue_config in queues_list.items(): sort_queue_name[queue_name] = len(ztq_core.get_error_queue(queue_name)) for queue_name in sorted(sort_queue_name, key=lambda x: sort_queue_name[x], reverse=True): task_queue = {} task_queue['name'] = queue_name #task_queue['tags'] = queue_config.get('tags',()) queue = ztq_core.get_task_queue(queue_name) # 任务数/错误数 task_queue['length'] = len(queue) task_queue['error_length'] = sort_queue_name[queue_name] #任务首个时间 task_queue['error_end'] = task_queue['first'] = '' first_job = queue[0] first_job= ztq_core.get_task_hash(queue_name).get(first_job) if first_job: task_queue['first'] = datetime.datetime.fromtimestamp(first_job['runtime'].get('create', 0)) #错误最末一个的时间 error_first_job = ztq_core.get_error_queue(queue_name)[0] error_first_job = ztq_core.get_error_hash(queue_name).get(error_first_job) if error_first_job: task_queue['error_end'] = datetime.datetime.fromtimestamp(error_first_job['runtime'].get('create', 0)) task_queue['weight'] = queue_weight.get(queue_name, 0) # 获取worker工作线程配置 workers_config = ztq_core.get_worker_config() task_queue['from_right'] = True for worker_name,worker_config in workers_config.items(): task_queue['workers'] = [] for config in worker_config.get(queue_name,[]): task_queue['workers'].append([worker_name+':', config['interval']]) if 'from_right' in config: task_queue['from_right'] = config['from_right'] task_queue['buffer_length'] = len(ztq_core.get_buffer_queue(queue_name)) yield task_queue
def run(self): if not self.config: return while not self._stop: for buffer_name, buffer_config in self.config.items(): # 需要停止 if self._stop: return self.buffer_queue = ztq_core.get_buffer_queue(buffer_name) self.task_queue = ztq_core.get_task_queue(buffer_name) self.buffer_name = buffer_name self.task_queue_limit = int(buffer_config['thread_limit']) while True: try: self.start_job() break except ztq_core.ConnectionError: time.sleep(3) time.sleep(1)