def test(self,
key,
params=None,
params_path=None,
sub_key=None,
report_tag=None,
report_scene=None,
debug=True):
if self.is_running is True: # 一旦运行起来，不可再进入test模式即调用test方法
raise RuntimeError("Can not test, current is running")
self.debug = debug
if params is None and params_path is not None:
with open(params_path, "r") as rp:
c = rp.read()
params = json.loads(c)
task_item = WorkerTask(work_tag=self.work_tag,
task_key=key,
task_sub_key=sub_key,
task_report_tag=report_tag)
if report_scene is not None:
task_item.set(task_report_scene=report_scene)
if self.expect_params_type is not None:
if not isinstance(params, self.expect_params_type):
raise TypeError("params should", self.expect_params_type)
if isinstance(params, dict):
task_item.set(task_params=WorkerTaskParams(**params))
task_item.task_params.debug_func = self.task_debug_log
else:
task_item.set(task_params=params)
if StringTool.is_string(self.log_dir) is True:
log_name = StringTool.join_encode(
[self.work_tag, "_", task_item.task_key, ".log"], join_str="")
task_item.log_path = StringTool.path_join(self.log_dir, log_name)
self.current_task = task_item
return self._execute()
def hang_up_clock(self, freq=None):
# test模式下或者还没有运行，is_running为False不进行打卡
if self.is_running is False:
return
loop_run = True
if isinstance(freq, int) and freq >= 1:
loop_run = False
else:
freq = 0
key = self.clock_key
hang_freq = 0
while self.is_running:
# run以后才启动线程，所以此判断可以不用了
# if self.is_running is False and loop_run is True:
# time.sleep(5)
# continue
try:
if self.current_task is not None and self.current_task.task_key is not None:
v = StringTool.join([
self.heartbeat_value,
int(time.time()), self.current_task.task_key
], "_").strip("_")
else:
v = StringTool.join(
[self.heartbeat_value,
int(time.time())], "_").strip("_")
self.redis_man.setex(key, v, 60)
except RedisError:
pass
hang_freq += 1
if hang_freq < freq or loop_run is True:
time.sleep(55)
else:
break
def remove_queue_task(self, work_tag, key, report_tag=None, sub_key=None):
task_key = key
if report_tag is not None:
re_work_tag = StringTool.join_decode([work_tag, report_tag],
join_str="|")
else:
re_work_tag = work_tag
if sub_key is not None:
key = StringTool.join_decode([key, sub_key], join_str="|")
value_prefix = StringTool.join_decode([re_work_tag, key], ",")
value_prefix2 = RedisQueue.package_task_v2(work_tag,
task_key,
"",
sub_key=sub_key,
return_prefix=True)
queue_tasks = self.list_queue_detail(work_tag)
if queue_tasks is None:
return 0
count = 0
key = StringTool.join_decode([self.queue_prefix_key, work_tag],
join_str="_")
for task in queue_tasks:
if task.startswith(value_prefix) is True or task.startswith(
value_prefix2) is True:
try:
count += self.redis_man.lrem(key, task, num=0)
except Exception:
continue
return count
def write_example(self, work_tag, params):
save_name = StringTool.join_decode([
self.current_task.task_key, self.current_task.task_sub_key,
int(time.time()), "json"
],
join_str=".")
save_dir = StringTool.path_join(example_dir, work_tag)
if os.path.isdir(save_dir) is False:
os.mkdir(save_dir)
save_path = StringTool.path_join(save_dir, save_name)
with open(save_path, "w") as wp:
wp.write(StringTool.encode(json.dumps(params)))
return save_path
def parse(cls, s):
if isinstance(s, TaskStatus):
return s
if StringTool.is_string(s) is False:
return None
for key, value in cls.__dict__.items():
if StringTool.is_string(value) is False:
continue
if key.startswith("_") is True:
continue
if isinstance(value, SimpleString) is False:
continue
if value.lower() == s.lower():
return value
return None
def pop_task(self, freq=0):
self.num_pop_task += 1
next_task = None
try:
if self.num_pop_task % 5 != 0:
tasks = self.redis_man.blpop(self.queue_key, self.pop_time_out)
if tasks is not None:
next_task = tasks[1]
else:
tasks = self.redis_man.zrangebyscore(self.delay_queue_key, 0,
time.time())
for item in tasks:
l = self.redis_man.zrem(self.delay_queue_key, item)
if l > 0:
next_task = item
break
except Exception as e:
if freq > 5:
self.worker_log("[REDIS POP ERROR MSG]", e, level="ERROR")
raise e
time.sleep(5 * freq + 10)
return self.pop_task(freq=freq + 1)
if next_task is not None:
t = StringTool.decode(next_task)
return t
return next_task
def __init__(self, log_dir=None, work_tag=None, **kwargs):
WorkerConfig.__init__(self, work_tag=work_tag, **kwargs)
_WorkerLog.__init__(self, log_dir=log_dir, **kwargs)
if StringTool.is_string(self.work_tag) is False:
class_name = self.__class__.__name__
msg = "Need String work_tag. Please Set {0}.DEFAULT_WORK_TAG=yourWorkTag Or {0}(work_tag=yourWorkTag)"
raise TypeError(msg.format(class_name))
if ValueVerify.v_work_tag(self.work_tag) is False:
raise ValueError("Invalid work_tag format")
self._id = uuid.uuid4().hex # add in 0.9.11
self._msg_manager = None
self.is_running = False # 表示worker是否已经开始运行，并不断接收任务,一旦运行起来，不可再进入test模式即调用test方法
self._debug = False
self.before_handle_funcs = []
self.after_handle_funcs = []
self.init_log_dir()
self._handle_task_func = self.handle_task
self.num_success_job = 0 # add in 0.8.1
self.num_fail_job = 0 # add in 0.8.1
self.num_wrongful_job = 0 # add in 0.8.1
self.num_invalid_job = 0 # add in 0.8.1
self.num_null_job = 0 # add in 0.8.1
self.num_pop_task = 0 # add in 1.6.8 尝试去获得任务的次数（无论是否获得数据，无论从哪个队列中获得）
if "worker_index" in kwargs:
self.worker_index = kwargs["worker_index"]
if "redirect_stdout" in kwargs:
self.redirect_stdout = kwargs["redirect_stdout"]
self.heartbeat_key = self.heartbeat_prefix_key + "_" + self.work_tag
self.queue_key = self.queue_prefix_key + "_" + self.work_tag
# 延时队列，该队列和普通queue相对，访问一定次数的普通queue才会访问一次该队列
self.delay_queue_key = self.queue_prefix_key + "_" + self.work_tag + "@delay"
self.clock_key = self.clock_prefix_key + "_" + self.work_tag + "_" + self._id
self.current_task = WorkerTask()
self._worker_status = 0 # 内部运行状态。目前主要用于 当收到kill信号时的处理
def list_heartbeat_detail(self, work_tag):
"""
add in version 1.0.6
"""
key = StringTool.join_encode(
[self.heartbeat_prefix_key, "_", work_tag])
print(key)
return self.redis_man.get(key)
def has_heartbeat(self):
current_value = StringTool.decode(
self.redis_man.get(self.heartbeat_key))
if current_value != self.heartbeat_value:
self.worker_log("heartbeat is", self.heartbeat_value, "now is",
current_value)
return False
return True
def _set_report_scene(self, report_scene):
if is_number(report_scene) is False:
try:
if StringTool.is_string(report_scene) is False:
return None
report_scene = int(report_scene)
except ValueError:
return None
self.task_report_scene = report_scene
def getpath(self, k, d=None):
"""
add in version 1.4.5
"""
v = self.get(k, d)
v = StringTool.encode(v)
if os.path.exists(v) is False:
raise WorkerTaskParamsValueTypeError(k, v, "path")
return v
def clear_task_item(self, work_tag, key):
k_l = [self.queue_prefix_key, work_tag, key, "*"]
key_prefix = StringTool.join_decode(k_l, "_")
hs = self.redis_man.keys(key_prefix)
task_items = dict(sub=dict(), values=dict())
for item in hs:
if self.redis_man.type(item) != "hash":
continue
self.redis_man.delete(item)
return task_items
def look_task_item():
arg_man.add_argument("-w", "--work-tag", dest="work_tag", help="work tag", metavar="")
args = parse_args()
rs = RedisStat()
values = []
if args.work_tag is not None:
values.append(args.work_tag)
while True:
prompt_prefix = ""
if len(values) <= 0:
work_tag = jy_input("Please Input Work_tag", prompt_prefix=prompt_prefix)
work_tag = work_tag.strip()
if work_tag.lower() in ("e", "exit"):
sys.exit(0)
values.append(work_tag)
continue
if len(values) == 1:
prompt_prefix += "[work_tag:%s]" % values[0]
key = jy_input("Please Input Task Key", prompt_prefix=prompt_prefix)
key = key.strip()
if key.lower() in ("e", "exit"):
values.remove(values[-1])
continue
values.append(key)
continue
elif len(values) >= 2:
sub_key = None
prompt_prefix += "[work_tag:%s][key:%s]" % (values[0], values[1])
if len(values) >= 3:
sub_key = StringTool.join_decode(values[2:], "_")
prompt_prefix += "[sub_key:%s]" % sub_key
task_items = rs.list_task_item(values[0], values[1], sub_key)
prompt = ""
if len(task_items["sub"].keys()) > 0:
prompt += "Input follow value look sub item\n"
prompt += "\n".join(task_items["sub"])
prompt += "\n"
if len(task_items["values"].keys()) > 0:
prompt += "Input follow value look item value\n"
prompt += "\n".join(task_items["values"])
while True:
item_key = jy_input(prompt, prompt_prefix=prompt_prefix)
item_key = item_key.strip()
if item_key.lower() in ("e", "exit"):
values.remove(values[-1])
break
if item_key in task_items["sub"]:
values.append(item_key)
break
elif item_key in task_items["values"]:
print(task_items["values"][item_key])
continue
else:
continue
def _handle_package_sub_part(cls, data, action="package"):
if action == "package":
if isinstance(data, (tuple, list)):
p = StringTool.join_decode(
map(lambda x: cls.sub_part_handler.escape(x), data), "|")
else:
p = cls.sub_part_handler.escape("%s" % data)
return p
elif action == "unpack":
s_data = data.split("|")
return map(cls.sub_part_handler.unescape, s_data)
return None
def list_heartbeat(self):
"""
add in version 1.0.6
"""
l_h = []
key_prefix = StringTool.join_decode([self.heartbeat_prefix_key, "_*"])
len_k = len(key_prefix) - 1
hs = self.redis_man.keys(key_prefix)
for item in hs:
if self.redis_man.type(item) == "string":
tag = item[len_k:]
if len(tag) > 0:
l_h.append(tag)
return l_h
def list_task_item(self, work_tag, key, sub_key=None):
k_l = [self.queue_prefix_key, work_tag, key]
if sub_key is not None:
k_l.append(sub_key)
task_item_compile = re.compile(
re.escape(StringTool.join_decode(k_l, "_")) + "_(\\d+)$")
get_key = StringTool.join_decode([k_l], "_")
k_l.append("*")
key_prefix = StringTool.join_decode(k_l, "_")
hs = self.redis_man.keys(key_prefix)
task_items = dict(sub=dict(), values=dict())
for item in hs:
if self.redis_man.type(item) != "hash":
continue
m_r = task_item_compile.match(item)
if m_r is None:
continue
task_items["sub"][m_r.groups()[0]] = item
if self.redis_man.type(get_key) == "hash":
item = self.redis_man.hgetall(get_key)
for key in item.keys():
task_items["values"][key] = StringData.unpack_data(item[key])
return task_items
def worker_log(self, *args, **kwargs):
if self.log_dir is None or is_string(self.log_dir) is False:
return
msg = StringTool.join(args, " ")
level = kwargs.pop("level", "INFO")
level = str(level).upper()
if level not in ["INFO", "DEBUG"]:
self.publish_message(msg)
log_file = os.path.join(self.log_dir, "%s.log" % self.work_tag)
now_time = datetime.now().strftime(TIME_FORMAT)
write_a = ["[", self.heartbeat_value]
if self.worker_index is not None:
write_a.extend([":", self.worker_index])
write_a.extend(["] ", now_time, ": ", level, " ", msg, "\n"])
with open(log_file, "ab", 0) as wl:
u = StringTool.join(write_a, join_str="")
s = StringTool.encode(u)
wl.write(s)
if self.redirect_stdout is False and self.debug is True:
try:
logging.info(s)
except Exception as e:
pass
def list_worker_detail(self, work_tag):
"""
add in version 0.9.7
"""
d_wd = dict()
work_tag = StringTool.encode(work_tag)
key = StringTool.join([self.clock_prefix_key, work_tag, "*"],
"_").strip("_")
len_k = len(self.clock_prefix_key) + 2 + len(work_tag)
ws = self.redis_man.keys(key)
for item in ws:
if self.redis_man.type(item) != "string":
continue
pre_key = item[len_k:]
if re.search(r"[^\da-z]", pre_key, re.I) is not None:
continue
p = dict()
v = self.redis_man.get(item)
p["value"] = v
vs = v.split("_", 2)
if len(vs) < 2:
continue
p["heartbeat_value"] = vs[0]
p["clock_time"] = vs[1]
try:
p["clock_time"] = int(p["clock_time"])
x = time.localtime(p["clock_time"])
p["clock_time2"] = time.strftime("%Y-%m-%d %H:%M:%S", x)
except ValueError:
pass
if len(vs) > 2:
p["current_task"] = vs[2]
p["working"] = True
else:
p["working"] = False
d_wd[pre_key] = p
return d_wd
def task_log(self, *args, **kwargs):
if self.current_task is None or self.current_task.log_path is None:
return
msg = StringTool.join(args, " ")
level = kwargs.pop("level", "INFO")
level = str(level).upper()
if level not in ["INFO", "DEBUG"]:
p_msg_a = [self.current_task.task_key]
if self.current_task.task_sub_key is not None:
p_msg_a.extend([" ", self.current_task.task_sub_key])
p_msg = StringTool.join([p_msg_a, "\n", msg], "")
self.publish_message(p_msg)
if self.upload_log_tag is not None:
upload_info = dict(log_path=self.current_task.log_path,
timestamp=int(time.time()))
self.push_task(StringTool.join_decode(
[self.current_task.task_key, self.work_tag], join_str="_"),
upload_info,
work_tag=self.upload_log_tag)
log_file = self.current_task.log_path
now_time = datetime.now().strftime(TIME_FORMAT)
write_a = ["[", self.heartbeat_value]
if self.worker_index is not None:
write_a.extend([":", self.worker_index])
if self.current_task.task_sub_key is not None:
write_a.extend(["][", self.current_task.task_sub_key])
write_a.extend(["] ", now_time, ": ", level, " ", msg, "\n"])
with open(log_file, "ab", 0) as wl:
u = StringTool.join(write_a, join_str="")
s = StringTool.encode(u)
wl.write(s)
if self.redirect_stdout is False and self.debug is True:
try:
logging.info(s)
except Exception as e:
pass
def get_dirty_item(self, work_tag):
k_l = [self.queue_prefix_key, work_tag, "*"]
key_prefix = StringTool.join_decode(k_l, "_")
prefix_len = len(key_prefix) - 1
hs = self.redis_man.keys(key_prefix)
all_keys = dict()
find_sub_key = re.compile("_(\d+)$")
for item in hs:
if self.redis_man.type(item) != "hash":
continue
search_r = find_sub_key.search(item)
if search_r is None:
continue
sub_key = search_r.groups()[0]
p = item[:0 - len(sub_key) - 1]
if p in all_keys:
all_keys[p].append(sub_key)
else:
all_keys[p] = [sub_key]
delete_items = []
# 删除 没有任务描述的零散任务
for key in all_keys.keys():
union_key = key[prefix_len:]
if "0" not in all_keys[key]:
delete_items.append(
dict(prefix=union_key,
sub_keys=all_keys[key],
message="未发现pipeline信息"))
continue
task_len = StringData.unpack_data(
self.redis_man.hget(key + "_0", "task_len"))
if task_len is None:
delete_items.append(
dict(prefix=union_key,
sub_keys=all_keys[key],
message="pipeline信息未发现task_len"))
continue
for i in range(task_len):
if "%s" % i not in all_keys[key]:
delete_items.append(
dict(prefix=union_key,
sub_keys=all_keys[key],
message="缺少子任务%s的信息" % i))
return delete_items
def write_pbs_task(self, work_tag, cmd):
save_name = StringTool.join_decode([
self.current_task.task_key, self.current_task.task_sub_key,
int(time.time()), "pbs"
],
join_str=".")
save_dir = StringTool.path_join(pbs_task_dir, work_tag)
if os.path.isdir(save_dir) is False:
os.mkdir(save_dir)
save_path = StringTool.path_join(save_dir, save_name)
with open(save_path, "w") as wp:
cmd = StringTool.join_encode(cmd, join_str=" ")
s = StringTool.join_encode([pbs_template, cmd], join_str="\n")
wp.write(StringTool.encode(s))
return save_path
def __init__(self,
conf_path=None,
heartbeat_value=None,
is_brother=False,
work_tag=None,
log_dir=None,
redis_host=None,
redis_password=None,
redis_port=None,
redis_db=None,
section_name="Redis",
**kwargs):
self.conf_path = conf_path
if self.conf_path is None or is_string(
self.conf_path) is False or os.path.exists(
self.conf_path) is False:
logging.debug("Conf Path %s Not Exist ", self.conf_path)
logging.debug("Read os environ : %s", self.conf_path_environ_key)
env_conf_path = os.environ.get(self.conf_path_environ_key)
logging.debug("os environ %s %s", self.conf_path_environ_key,
env_conf_path)
if env_conf_path is not None:
if os.path.exists(env_conf_path) is True:
self.conf_path = env_conf_path
logging.debug("Use %s As conf path", env_conf_path)
else:
logging.debug("Path %s Not Exist", env_conf_path)
RedisWorkerConfig.__init__(self,
self.conf_path,
redis_host=redis_host,
redis_password=redis_password,
redis_port=redis_port,
redis_db=redis_db,
section_name=section_name)
Worker.__init__(self,
conf_path=self.conf_path,
work_tag=work_tag,
log_dir=log_dir,
**kwargs)
self.stat_man = RedisStat(conf_path=self.conf_path,
redis_host=redis_host,
redis_password=redis_password,
redis_port=redis_port,
redis_db=redis_db,
section_name=section_name)
if is_brother is True:
current_heartbeat = self.redis_man.get(self.heartbeat_key)
if current_heartbeat is not None:
heartbeat_value = current_heartbeat
if heartbeat_value is None:
heartbeat_value = StringTool.random_str(str_len=12, upper_s=False)
self.heartbeat_value = StringTool.decode(heartbeat_value)
if ValueVerify.v_heartbeat(self.heartbeat_value) is False:
raise ValueError(
"heartbeat only allow 0-9 a-z and length between 3 and 50.")
self.t_clock = threading.Thread(target=self.hang_up_clock)
self.t_clock.daemon = True # 主进程退出时，随主进程一起退出
if "upload_log_tag" in kwargs:
self.upload_log_tag = kwargs["upload_log_tag"]
else:
self.upload_log_tag = None
def write(self, s):
self._w.write(StringTool.encode(s))
def read_task_log(self,
work_tag,
key,
sub_key=None,
sub_key_prefix=None,
level="INFO",
max_length=1000000):
"""
:param work_tag:
:param key:
:param sub_key: 为None时查询所有有子key和无子key的日志，为空字符串时仅查询无子key的日志，为具体某个子key时查询具体子key的日志
:param level: 默认为INFO，允许DEBUG，INFO，WARNING，ERROR。其他值认为是INFO
:return:
"""
name = StringTool.join([work_tag, "_", key, ".log"], "")
log_path = StringTool.path_join(self.log_dir, work_tag.lower(), name)
if os.path.exists(log_path) is False:
log_path = StringTool.path_join(self.log_dir, name)
if os.path.exists(log_path) is False:
return False, None
s_log = os.stat(log_path)
read_seek = s_log.st_size - max_length if max_length < s_log.st_size else 0
# 处理参数
if sub_key is not None:
sub_key = StringTool.encode(sub_key)
if sub_key_prefix is not None:
sub_key_prefix = StringTool.encode(sub_key_prefix)
if StringTool.is_string(level) is False:
level = "INFO"
level = level.upper()
if level not in self.log_level:
level = "INFO"
allow_levels = self.log_level[level]
logs_list = []
last_save = False
with open(log_path, "r") as rl:
rl.seek(read_seek)
c = rl.read()
all_lines = c.split("\n")
for line in all_lines:
rl = self.log_compile.match(line)
if rl is not None:
line_sub_key = rl.groups()[0]
log_time = rl.groups()[1]
if len(line_sub_key) >= 2:
line_sub_key = line_sub_key[1:-1]
line_level = rl.groups()[2]
log_msg = rl.groups()[3]
if sub_key is not None and sub_key != line_sub_key:
last_save = False
continue
if sub_key_prefix is not None and line_sub_key.startswith(
sub_key_prefix) is False:
last_save = False
continue
if line_level not in allow_levels:
last_save = False
continue
last_save = True
logs_list.append(
map(StringTool.decode,
[line_sub_key, log_time, line_level, log_msg]))
elif last_save is True:
logs_list[-1][3] = StringTool.join_decode(
[logs_list[-1][3], line])
return True, logs_list
def write_line(self, s):
self._w.write(StringTool.encode(s) + "\n")
def write_array(self, a, connector="\t"):
s = StringTool.join_encode(a, join_str=connector)
self.write_line(s)
def __init__(self, file_path, mode="w"):
self.file_path = StringTool.encode(file_path)
self.mode = mode
self._w = None
self._w = open(self.file_path, mode=self.mode)
import sys
import os
import time
import json
import uuid
import tempfile
import ConfigParser
from JYTools import StringTool
from JYTools.JYWorker import RedisWorker, worker_run
__author__ = '鹛桑够'
sys_tmp_dir = tempfile.gettempdir()
log_dir = os.environ.get("JINGD_LOG_DIR", sys_tmp_dir)
agent_dir = StringTool.path_join(log_dir, "pbs_agent")
if os.path.isdir(agent_dir) is False:
os.mkdir(agent_dir)
example_dir = StringTool.path_join(agent_dir, "example")
pbs_task_dir = StringTool.path_join(agent_dir, "pbs")
pbs_log_dir = StringTool.path_join(agent_dir, "log")
if os.path.isdir(example_dir) is False:
os.mkdir(example_dir)
if os.path.isdir(pbs_task_dir) is False:
os.mkdir(pbs_task_dir)
if os.path.isdir(pbs_log_dir) is False:
os.mkdir(pbs_log_dir)
pbs_template = """#PBS -S /bin/bash
#PBS -m n
#PBS -M <*****@*****.**>
def delete_heartbeat(self, work_tag):
key = StringTool.join_encode(
[self.heartbeat_prefix_key, "_", work_tag])
return self.redis_man.delete(key)
def parse_task_info(self, task_info):
task_item = WorkerTask(task_info=task_info)
if task_info.startswith("$2") is True:
un_r, data = RedisQueue.unpack_task(task_info)
if un_r is False:
return False, data
if len(data["key"]) <= 0:
return True, None
task_item.set(**data)
if isinstance(task_item.task_params, WorkerTaskParams):
task_item.task_params.debug_func = self.task_debug_log
else:
self.worker_log("handle old data format")
partition_task = task_info.split(",", 3)
if len(partition_task) != 4:
error_msg = "Invalid task %s, task partition length is not 3" % task_info
return False, error_msg
work_tags = partition_task[0].split("|") # 0 work tag 1 return tag
if work_tags[0] != self.work_tag:
error_msg = "Invalid task %s, task not match work tag %s" % (
task_info, self.work_tag)
return False, error_msg
task_item.set(work_tag=work_tags[0])
if len(work_tags) > 1:
task_item.set(task_report_tag=work_tags[1])
keys = partition_task[1].split("|")
if len(keys[0]) <= 0:
return True, None
task_item.set(task_key=keys[0])
if len(keys) > 1:
task_item.set(task_sub_key=keys[1])
if partition_task[2] not in ("string", "json", "report",
"control"):
error_msg = "Invalid task %s, task args type invalid" % task_info
return False, error_msg
params = partition_task[3]
if partition_task[2] in ("json", "report", "control"):
try:
params = json.loads(params)
except ValueError:
error_msg = "Invalid task %s, task args type and args not uniform" % task_info
return False, error_msg
if partition_task[2] == "report":
task_item.set(task_type=TaskType.Report)
task_item.set(task_params=WorkerTask(**params))
elif partition_task[2] == "control":
task_item.set(task_type=TaskType.Control)
if "expected_status" not in params:
return False, "Invalid Task, not found expected_status in params"
expected_status = TaskStatus.parse(params["expected_status"])
if expected_status is None:
return False, "Invalid Task, unknown expected status, %s" % params[
"expected_status"]
task_item.set(task_params=WorkerTaskParams(**params))
task_item.task_params.debug_func = self.task_debug_log
else:
if self.expect_params_type is not None:
if not isinstance(params, self.expect_params_type):
return False, "Invalid task, not expect param type"
if isinstance(self.expect_params_type, dict) is True:
task_item.set(task_params=WorkerTaskParams(**params))
task_item.task_params.debug_func = self.task_debug_log
else:
task_item.set(task_params=params)
if StringTool.is_string(self.log_dir) is True:
log_name = StringTool.join_encode(
[self.work_tag, "_", task_item.task_key, ".log"], join_str="")
task_item.log_path = StringTool.path_join(self.log_dir, log_name)
return True, task_item