def pop_task(self, freq=0):
self.num_pop_task += 1
next_task = None
try:
if self.num_pop_task % 5 != 0:
tasks = self.redis_man.blpop(self.queue_key, self.pop_time_out)
if tasks is not None:
next_task = tasks[1]
else:
tasks = self.redis_man.zrangebyscore(self.delay_queue_key, 0,
time.time())
for item in tasks:
l = self.redis_man.zrem(self.delay_queue_key, item)
if l > 0:
next_task = item
break
except Exception as e:
if freq > 5:
self.worker_log("[REDIS POP ERROR MSG]", e, level="ERROR")
raise e
time.sleep(5 * freq + 10)
return self.pop_task(freq=freq + 1)
if next_task is not None:
t = StringTool.decode(next_task)
return t
return next_task
def has_heartbeat(self):
current_value = StringTool.decode(
self.redis_man.get(self.heartbeat_key))
if current_value != self.heartbeat_value:
self.worker_log("heartbeat is", self.heartbeat_value, "now is",
current_value)
return False
return True
def __init__(self,
conf_path=None,
heartbeat_value=None,
is_brother=False,
work_tag=None,
log_dir=None,
redis_host=None,
redis_password=None,
redis_port=None,
redis_db=None,
section_name="Redis",
**kwargs):
self.conf_path = conf_path
if self.conf_path is None or is_string(
self.conf_path) is False or os.path.exists(
self.conf_path) is False:
logging.debug("Conf Path %s Not Exist ", self.conf_path)
logging.debug("Read os environ : %s", self.conf_path_environ_key)
env_conf_path = os.environ.get(self.conf_path_environ_key)
logging.debug("os environ %s %s", self.conf_path_environ_key,
env_conf_path)
if env_conf_path is not None:
if os.path.exists(env_conf_path) is True:
self.conf_path = env_conf_path
logging.debug("Use %s As conf path", env_conf_path)
else:
logging.debug("Path %s Not Exist", env_conf_path)
RedisWorkerConfig.__init__(self,
self.conf_path,
redis_host=redis_host,
redis_password=redis_password,
redis_port=redis_port,
redis_db=redis_db,
section_name=section_name)
Worker.__init__(self,
conf_path=self.conf_path,
work_tag=work_tag,
log_dir=log_dir,
**kwargs)
self.stat_man = RedisStat(conf_path=self.conf_path,
redis_host=redis_host,
redis_password=redis_password,
redis_port=redis_port,
redis_db=redis_db,
section_name=section_name)
if is_brother is True:
current_heartbeat = self.redis_man.get(self.heartbeat_key)
if current_heartbeat is not None:
heartbeat_value = current_heartbeat
if heartbeat_value is None:
heartbeat_value = StringTool.random_str(str_len=12, upper_s=False)
self.heartbeat_value = StringTool.decode(heartbeat_value)
if ValueVerify.v_heartbeat(self.heartbeat_value) is False:
raise ValueError(
"heartbeat only allow 0-9 a-z and length between 3 and 50.")
self.t_clock = threading.Thread(target=self.hang_up_clock)
self.t_clock.daemon = True # 主进程退出时，随主进程一起退出
if "upload_log_tag" in kwargs:
self.upload_log_tag = kwargs["upload_log_tag"]
else:
self.upload_log_tag = None
