def package_task_info(work_tag,
key,
params,
sub_key=None,
report_tag=None,
report_scene=ReportScene.END,
is_report=False):
"""
info format: work_tag[|report_tag[:report_scene]],key[|sub_key],args_type(task_type),args
args_type: json
example: jy_task,key_1,json,{"v":1}
example: jy_task|ping,key_1|first,json,{"v":1}
"""
if sub_key is not None:
key = "%s|%s" % (key, sub_key)
if is_string(work_tag) is False:
raise InvalidWorkTag()
if ValueVerify.v_work_tag(work_tag) is False:
raise InvalidWorkTag()
if report_tag is not None:
if ValueVerify.v_report_tag(report_tag) is False:
raise InvalidWorkTag()
work_tag = "%s|%s:%s" % (work_tag, report_tag, report_scene)
v = "%s,%s," % (work_tag, key)
if isinstance(params, dict):
if is_report is False:
v += "json," + json.dumps(params)
else:
v += "report," + json.dumps(params)
else:
v += "string," + params
return v
def join_decode(a, join_str=""):
join_str = decode(join_str)
if is_string(a):
r_a = decode(a)
elif isinstance(a, (tuple, list)):
a_tmp = map(lambda x: join_decode(x, join_str), a)
r_a = join_str.join(a_tmp)
else:
r_a = str(a)
return r_a
def join(a, join_str):
r_a = ""
if is_string(a):
r_a += decode(a) + join_str
elif isinstance(a, (tuple, list)):
for item in a:
r_a += join(item, join_str)
else:
r_a += decode(str(a)) + join_str
return r_a
def package_task(work_tag,
key,
params,
sub_key=None,
task_type=TaskType.Normal,
**kwargs):
"""
info format: work_tag[|report_tag[:report_scene]],key[|sub_key],args_type(task_type),args
args_type: json
example: jy_task,key_1,json,{"v":1}
example: jy_task|ping,key_1|first,json,{"v":1}
"""
if sub_key is not None:
key = "%s|%s" % (key, sub_key)
if is_string(work_tag) is False:
raise InvalidWorkTag()
if ValueVerify.v_work_tag(work_tag) is False:
raise InvalidWorkTag()
if task_type == TaskType.Normal:
report_tag = kwargs.pop("report_tag", None)
if report_tag is not None:
if ValueVerify.v_report_tag(report_tag) is False:
raise InvalidWorkTag()
report_scene = kwargs.pop("report_scene", ReportScene.END)
work_tag = "%s|%s:%s" % (work_tag, report_tag, report_scene)
if isinstance(params, dict):
args_s = "json," + json.dumps(params)
else:
args_s = "string," + json.dumps(params)
elif task_type == TaskType.Control:
if "expected_status" not in params:
raise RuntimeError("Not found expected_status in params")
expected_status = TaskStatus.parse(params["expected_status"])
if expected_status is None:
raise RuntimeError("Invalid expected_status")
params["expected_status"] = expected_status
args_s = "control," + json.dumps(params)
elif task_type == TaskType.Report:
args_s = "report," + json.dumps(params)
else:
raise RuntimeError("Invalid task type")
v = "%s,%s,%s" % (work_tag, key, args_s)
return v
def worker_log(self, *args, **kwargs):
if self.log_dir is None or is_string(self.log_dir) is False:
return
msg = StringTool.join(args, " ")
level = kwargs.pop("level", "INFO")
level = str(level).upper()
if level not in ["INFO", "DEBUG"]:
self.publish_message(msg)
log_file = os.path.join(self.log_dir, "%s.log" % self.work_tag)
now_time = datetime.now().strftime(TIME_FORMAT)
write_a = ["[", self.heartbeat_value]
if self.worker_index is not None:
write_a.extend([":", self.worker_index])
write_a.extend(["] ", now_time, ": ", level, " ", msg, "\n"])
with open(log_file, "ab", 0) as wl:
u = StringTool.join(write_a, join_str="")
s = StringTool.encode(u)
wl.write(s)
if self.redirect_stdout is False and self.debug is True:
try:
logging.info(s)
except Exception as e:
pass
def __init__(self,
conf_path=None,
heartbeat_value=None,
is_brother=False,
work_tag=None,
log_dir=None,
redis_host=None,
redis_password=None,
redis_port=None,
redis_db=None,
section_name="Redis",
**kwargs):
self.conf_path = conf_path
if self.conf_path is None or is_string(
self.conf_path) is False or os.path.exists(
self.conf_path) is False:
logging.debug("Conf Path %s Not Exist ", self.conf_path)
logging.debug("Read os environ : %s", self.conf_path_environ_key)
env_conf_path = os.environ.get(self.conf_path_environ_key)
logging.debug("os environ %s %s", self.conf_path_environ_key,
env_conf_path)
if env_conf_path is not None:
if os.path.exists(env_conf_path) is True:
self.conf_path = env_conf_path
logging.debug("Use %s As conf path", env_conf_path)
else:
logging.debug("Path %s Not Exist", env_conf_path)
RedisWorkerConfig.__init__(self,
self.conf_path,
redis_host=redis_host,
redis_password=redis_password,
redis_port=redis_port,
redis_db=redis_db,
section_name=section_name)
Worker.__init__(self,
conf_path=self.conf_path,
work_tag=work_tag,
log_dir=log_dir,
**kwargs)
self.stat_man = RedisStat(conf_path=self.conf_path,
redis_host=redis_host,
redis_password=redis_password,
redis_port=redis_port,
redis_db=redis_db,
section_name=section_name)
if is_brother is True:
current_heartbeat = self.redis_man.get(self.heartbeat_key)
if current_heartbeat is not None:
heartbeat_value = current_heartbeat
if heartbeat_value is None:
heartbeat_value = StringTool.random_str(str_len=12, upper_s=False)
self.heartbeat_value = StringTool.decode(heartbeat_value)
if ValueVerify.v_heartbeat(self.heartbeat_value) is False:
raise ValueError(
"heartbeat only allow 0-9 a-z and length between 3 and 50.")
self.t_clock = threading.Thread(target=self.hang_up_clock)
self.t_clock.daemon = True # 主进程退出时，随主进程一起退出
if "upload_log_tag" in kwargs:
self.upload_log_tag = kwargs["upload_log_tag"]
else:
self.upload_log_tag = None
def package_task_v2(cls,
work_tag,
key,
params,
sub_key=None,
task_type=TaskType.Normal,
return_prefix=False,
**kwargs):
"""
add in 1.7.8 use to replace package_task and package_task_info
format:
$2,task_type,work_tag,key[|sub_key],report_tag[|report_scene],data_type|task_data
:param work_tag:
:param key:
:param params:
:param sub_key:
:param task_type:
:param kwargs:
:return:
"""
if "is_report" in kwargs and kwargs["is_report"] is True:
task_type = TaskType.Report
# part 0 version + part 1 task_type
ps = ["$2", task_type]
if is_string(work_tag) is False:
raise InvalidWorkTag()
if ValueVerify.v_work_tag(work_tag) is False:
raise InvalidWorkTag()
# part 2 work_tag
ps.append(work_tag)
if sub_key is not None:
key = ["%s" % key, "%s" % sub_key]
if "task_name" in kwargs and is_string(kwargs["task_name"]):
key.append(kwargs["task_name"])
# part 3 key
ps.append(key)
report_tag = kwargs.pop("report_tag", "")
if report_tag is not None and len(report_tag) > 0:
if ValueVerify.v_report_tag(report_tag) is False:
raise InvalidWorkTag()
report_scene = kwargs.pop("report_scene", ReportScene.END)
report_tag = (report_tag, "%s" % report_scene)
if report_tag is None:
report_tag = ""
# report tag
ps.append(report_tag)
if task_type == TaskType.Normal:
if isinstance(params, dict):
args_s = ["json", json.dumps(params)]
else:
args_s = ["str", str(params)]
elif task_type == TaskType.Control:
if "expected_status" not in params:
raise RuntimeError("Not found expected_status in params")
expected_status = TaskStatus.parse(params["expected_status"])
if expected_status is None:
raise RuntimeError("Invalid expected_status")
params["expected_status"] = expected_status
args_s = ["json", json.dumps(params)]
elif task_type == TaskType.Report:
args_s = ["json", json.dumps(params)]
else:
raise RuntimeError("Invalid task type")
ps.append(args_s)
if return_prefix is True:
v = ",".join(
map(
lambda x: cls.part_handler.escape(
cls._handle_package_sub_part(x)), ps[:-2]))
else:
v = ",".join(
map(
lambda x: cls.part_handler.escape(
cls._handle_package_sub_part(x)), ps))
return v