def parse(cls, s):
if isinstance(s, TaskStatus):
return s
if StringTool.is_string(s) is False:
return None
for key, value in cls.__dict__.items():
if StringTool.is_string(value) is False:
continue
if key.startswith("_") is True:
continue
if isinstance(value, SimpleString) is False:
continue
if value.lower() == s.lower():
return value
return None
def __init__(self, log_dir=None, work_tag=None, **kwargs):
WorkerConfig.__init__(self, work_tag=work_tag, **kwargs)
_WorkerLog.__init__(self, log_dir=log_dir, **kwargs)
if StringTool.is_string(self.work_tag) is False:
class_name = self.__class__.__name__
msg = "Need String work_tag. Please Set {0}.DEFAULT_WORK_TAG=yourWorkTag Or {0}(work_tag=yourWorkTag)"
raise TypeError(msg.format(class_name))
if ValueVerify.v_work_tag(self.work_tag) is False:
raise ValueError("Invalid work_tag format")
self._id = uuid.uuid4().hex # add in 0.9.11
self._msg_manager = None
self.is_running = False # 表示worker是否已经开始运行，并不断接收任务,一旦运行起来，不可再进入test模式即调用test方法
self._debug = False
self.before_handle_funcs = []
self.after_handle_funcs = []
self.init_log_dir()
self._handle_task_func = self.handle_task
self.num_success_job = 0 # add in 0.8.1
self.num_fail_job = 0 # add in 0.8.1
self.num_wrongful_job = 0 # add in 0.8.1
self.num_invalid_job = 0 # add in 0.8.1
self.num_null_job = 0 # add in 0.8.1
self.num_pop_task = 0 # add in 1.6.8 尝试去获得任务的次数（无论是否获得数据，无论从哪个队列中获得）
if "worker_index" in kwargs:
self.worker_index = kwargs["worker_index"]
if "redirect_stdout" in kwargs:
self.redirect_stdout = kwargs["redirect_stdout"]
self.heartbeat_key = self.heartbeat_prefix_key + "_" + self.work_tag
self.queue_key = self.queue_prefix_key + "_" + self.work_tag
# 延时队列，该队列和普通queue相对，访问一定次数的普通queue才会访问一次该队列
self.delay_queue_key = self.queue_prefix_key + "_" + self.work_tag + "@delay"
self.clock_key = self.clock_prefix_key + "_" + self.work_tag + "_" + self._id
self.current_task = WorkerTask()
self._worker_status = 0 # 内部运行状态。目前主要用于 当收到kill信号时的处理
def test(self,
key,
params=None,
params_path=None,
sub_key=None,
report_tag=None,
report_scene=None,
debug=True):
if self.is_running is True: # 一旦运行起来，不可再进入test模式即调用test方法
raise RuntimeError("Can not test, current is running")
self.debug = debug
if params is None and params_path is not None:
with open(params_path, "r") as rp:
c = rp.read()
params = json.loads(c)
task_item = WorkerTask(work_tag=self.work_tag,
task_key=key,
task_sub_key=sub_key,
task_report_tag=report_tag)
if report_scene is not None:
task_item.set(task_report_scene=report_scene)
if self.expect_params_type is not None:
if not isinstance(params, self.expect_params_type):
raise TypeError("params should", self.expect_params_type)
if isinstance(params, dict):
task_item.set(task_params=WorkerTaskParams(**params))
task_item.task_params.debug_func = self.task_debug_log
else:
task_item.set(task_params=params)
if StringTool.is_string(self.log_dir) is True:
log_name = StringTool.join_encode(
[self.work_tag, "_", task_item.task_key, ".log"], join_str="")
task_item.log_path = StringTool.path_join(self.log_dir, log_name)
self.current_task = task_item
return self._execute()
def _set_report_scene(self, report_scene):
if is_number(report_scene) is False:
try:
if StringTool.is_string(report_scene) is False:
return None
report_scene = int(report_scene)
except ValueError:
return None
self.task_report_scene = report_scene
def read_task_log(self,
work_tag,
key,
sub_key=None,
sub_key_prefix=None,
level="INFO",
max_length=1000000):
"""
:param work_tag:
:param key:
:param sub_key: 为None时查询所有有子key和无子key的日志，为空字符串时仅查询无子key的日志，为具体某个子key时查询具体子key的日志
:param level: 默认为INFO，允许DEBUG，INFO，WARNING，ERROR。其他值认为是INFO
:return:
"""
name = StringTool.join([work_tag, "_", key, ".log"], "")
log_path = StringTool.path_join(self.log_dir, work_tag.lower(), name)
if os.path.exists(log_path) is False:
log_path = StringTool.path_join(self.log_dir, name)
if os.path.exists(log_path) is False:
return False, None
s_log = os.stat(log_path)
read_seek = s_log.st_size - max_length if max_length < s_log.st_size else 0
# 处理参数
if sub_key is not None:
sub_key = StringTool.encode(sub_key)
if sub_key_prefix is not None:
sub_key_prefix = StringTool.encode(sub_key_prefix)
if StringTool.is_string(level) is False:
level = "INFO"
level = level.upper()
if level not in self.log_level:
level = "INFO"
allow_levels = self.log_level[level]
logs_list = []
last_save = False
with open(log_path, "r") as rl:
rl.seek(read_seek)
c = rl.read()
all_lines = c.split("\n")
for line in all_lines:
rl = self.log_compile.match(line)
if rl is not None:
line_sub_key = rl.groups()[0]
log_time = rl.groups()[1]
if len(line_sub_key) >= 2:
line_sub_key = line_sub_key[1:-1]
line_level = rl.groups()[2]
log_msg = rl.groups()[3]
if sub_key is not None and sub_key != line_sub_key:
last_save = False
continue
if sub_key_prefix is not None and line_sub_key.startswith(
sub_key_prefix) is False:
last_save = False
continue
if line_level not in allow_levels:
last_save = False
continue
last_save = True
logs_list.append(
map(StringTool.decode,
[line_sub_key, log_time, line_level, log_msg]))
elif last_save is True:
logs_list[-1][3] = StringTool.join_decode(
[logs_list[-1][3], line])
return True, logs_list
def parse_task_info(self, task_info):
task_item = WorkerTask(task_info=task_info)
if task_info.startswith("$2") is True:
un_r, data = RedisQueue.unpack_task(task_info)
if un_r is False:
return False, data
if len(data["key"]) <= 0:
return True, None
task_item.set(**data)
if isinstance(task_item.task_params, WorkerTaskParams):
task_item.task_params.debug_func = self.task_debug_log
else:
self.worker_log("handle old data format")
partition_task = task_info.split(",", 3)
if len(partition_task) != 4:
error_msg = "Invalid task %s, task partition length is not 3" % task_info
return False, error_msg
work_tags = partition_task[0].split("|") # 0 work tag 1 return tag
if work_tags[0] != self.work_tag:
error_msg = "Invalid task %s, task not match work tag %s" % (
task_info, self.work_tag)
return False, error_msg
task_item.set(work_tag=work_tags[0])
if len(work_tags) > 1:
task_item.set(task_report_tag=work_tags[1])
keys = partition_task[1].split("|")
if len(keys[0]) <= 0:
return True, None
task_item.set(task_key=keys[0])
if len(keys) > 1:
task_item.set(task_sub_key=keys[1])
if partition_task[2] not in ("string", "json", "report",
"control"):
error_msg = "Invalid task %s, task args type invalid" % task_info
return False, error_msg
params = partition_task[3]
if partition_task[2] in ("json", "report", "control"):
try:
params = json.loads(params)
except ValueError:
error_msg = "Invalid task %s, task args type and args not uniform" % task_info
return False, error_msg
if partition_task[2] == "report":
task_item.set(task_type=TaskType.Report)
task_item.set(task_params=WorkerTask(**params))
elif partition_task[2] == "control":
task_item.set(task_type=TaskType.Control)
if "expected_status" not in params:
return False, "Invalid Task, not found expected_status in params"
expected_status = TaskStatus.parse(params["expected_status"])
if expected_status is None:
return False, "Invalid Task, unknown expected status, %s" % params[
"expected_status"]
task_item.set(task_params=WorkerTaskParams(**params))
task_item.task_params.debug_func = self.task_debug_log
else:
if self.expect_params_type is not None:
if not isinstance(params, self.expect_params_type):
return False, "Invalid task, not expect param type"
if isinstance(self.expect_params_type, dict) is True:
task_item.set(task_params=WorkerTaskParams(**params))
task_item.task_params.debug_func = self.task_debug_log
else:
task_item.set(task_params=params)
if StringTool.is_string(self.log_dir) is True:
log_name = StringTool.join_encode(
[self.work_tag, "_", task_item.task_key, ".log"], join_str="")
task_item.log_path = StringTool.path_join(self.log_dir, log_name)
return True, task_item
if other.task_sub_key != self.task_sub_key:
return False
return True
def add_error_msg(self, *args):
for arg in args:
self.task_errors.append(arg)
def __str__(self):
return json.dumps(self.to_dict())
if __name__ == "__main__":
print(json.dumps({"s": SimpleString("s")}))
print(SimpleString("Abc") == "abc")
StringTool.is_string(SimpleString("ddd"))
ts = TaskStatus.parse("Stopping2")
print(ts)
print(TaskStatus.RUNNING)
print(TaskStatus.RUNNING == "Running")
print(TaskStatus.RUNNING == "RUNNING")
print(TaskStatus.RUNNING == "Running2")
print("Running" == TaskStatus.RUNNING)
print(TaskStatus.ERROR == TaskStatus.RUNNING)
print(TaskStatus.RUNNING == TaskStatus.RUNNING.value)
print(TaskStatus.is_running("ABD"))
import json
print(json.dumps({"expected_status": TaskStatus.STOPPED}))
wt = WorkerTask(task_type=TaskType.Report)