def pop_task(self, freq=0): self.num_pop_task += 1 next_task = None try: if self.num_pop_task % 5 != 0: tasks = self.redis_man.blpop(self.queue_key, self.pop_time_out) if tasks is not None: next_task = tasks[1] else: tasks = self.redis_man.zrangebyscore(self.delay_queue_key, 0, time.time()) for item in tasks: l = self.redis_man.zrem(self.delay_queue_key, item) if l > 0: next_task = item break except Exception as e: if freq > 5: self.worker_log("[REDIS POP ERROR MSG]", e, level="ERROR") raise e time.sleep(5 * freq + 10) return self.pop_task(freq=freq + 1) if next_task is not None: t = StringTool.decode(next_task) return t return next_task
def has_heartbeat(self): current_value = StringTool.decode( self.redis_man.get(self.heartbeat_key)) if current_value != self.heartbeat_value: self.worker_log("heartbeat is", self.heartbeat_value, "now is", current_value) return False return True
def __init__(self, conf_path=None, heartbeat_value=None, is_brother=False, work_tag=None, log_dir=None, redis_host=None, redis_password=None, redis_port=None, redis_db=None, section_name="Redis", **kwargs): self.conf_path = conf_path if self.conf_path is None or is_string( self.conf_path) is False or os.path.exists( self.conf_path) is False: logging.debug("Conf Path %s Not Exist ", self.conf_path) logging.debug("Read os environ : %s", self.conf_path_environ_key) env_conf_path = os.environ.get(self.conf_path_environ_key) logging.debug("os environ %s %s", self.conf_path_environ_key, env_conf_path) if env_conf_path is not None: if os.path.exists(env_conf_path) is True: self.conf_path = env_conf_path logging.debug("Use %s As conf path", env_conf_path) else: logging.debug("Path %s Not Exist", env_conf_path) RedisWorkerConfig.__init__(self, self.conf_path, redis_host=redis_host, redis_password=redis_password, redis_port=redis_port, redis_db=redis_db, section_name=section_name) Worker.__init__(self, conf_path=self.conf_path, work_tag=work_tag, log_dir=log_dir, **kwargs) self.stat_man = RedisStat(conf_path=self.conf_path, redis_host=redis_host, redis_password=redis_password, redis_port=redis_port, redis_db=redis_db, section_name=section_name) if is_brother is True: current_heartbeat = self.redis_man.get(self.heartbeat_key) if current_heartbeat is not None: heartbeat_value = current_heartbeat if heartbeat_value is None: heartbeat_value = StringTool.random_str(str_len=12, upper_s=False) self.heartbeat_value = StringTool.decode(heartbeat_value) if ValueVerify.v_heartbeat(self.heartbeat_value) is False: raise ValueError( "heartbeat only allow 0-9 a-z and length between 3 and 50.") self.t_clock = threading.Thread(target=self.hang_up_clock) self.t_clock.daemon = True # 主进程退出时,随主进程一起退出 if "upload_log_tag" in kwargs: self.upload_log_tag = kwargs["upload_log_tag"] else: self.upload_log_tag = None