def getpath(self, k, d=None):
"""
add in version 1.4.5
"""
v = self.get(k, d)
v = StringTool.encode(v)
if os.path.exists(v) is False:
raise WorkerTaskParamsValueTypeError(k, v, "path")
return v
def write_example(self, work_tag, params):
save_name = StringTool.join_decode([
self.current_task.task_key, self.current_task.task_sub_key,
int(time.time()), "json"
],
join_str=".")
save_dir = StringTool.path_join(example_dir, work_tag)
if os.path.isdir(save_dir) is False:
os.mkdir(save_dir)
save_path = StringTool.path_join(save_dir, save_name)
with open(save_path, "w") as wp:
wp.write(StringTool.encode(json.dumps(params)))
return save_path
def write_pbs_task(self, work_tag, cmd):
save_name = StringTool.join_decode([
self.current_task.task_key, self.current_task.task_sub_key,
int(time.time()), "pbs"
],
join_str=".")
save_dir = StringTool.path_join(pbs_task_dir, work_tag)
if os.path.isdir(save_dir) is False:
os.mkdir(save_dir)
save_path = StringTool.path_join(save_dir, save_name)
with open(save_path, "w") as wp:
cmd = StringTool.join_encode(cmd, join_str=" ")
s = StringTool.join_encode([pbs_template, cmd], join_str="\n")
wp.write(StringTool.encode(s))
return save_path
def worker_log(self, *args, **kwargs):
if self.log_dir is None or is_string(self.log_dir) is False:
return
msg = StringTool.join(args, " ")
level = kwargs.pop("level", "INFO")
level = str(level).upper()
if level not in ["INFO", "DEBUG"]:
self.publish_message(msg)
log_file = os.path.join(self.log_dir, "%s.log" % self.work_tag)
now_time = datetime.now().strftime(TIME_FORMAT)
write_a = ["[", self.heartbeat_value]
if self.worker_index is not None:
write_a.extend([":", self.worker_index])
write_a.extend(["] ", now_time, ": ", level, " ", msg, "\n"])
with open(log_file, "ab", 0) as wl:
u = StringTool.join(write_a, join_str="")
s = StringTool.encode(u)
wl.write(s)
if self.redirect_stdout is False and self.debug is True:
try:
logging.info(s)
except Exception as e:
pass
def list_worker_detail(self, work_tag):
"""
add in version 0.9.7
"""
d_wd = dict()
work_tag = StringTool.encode(work_tag)
key = StringTool.join([self.clock_prefix_key, work_tag, "*"],
"_").strip("_")
len_k = len(self.clock_prefix_key) + 2 + len(work_tag)
ws = self.redis_man.keys(key)
for item in ws:
if self.redis_man.type(item) != "string":
continue
pre_key = item[len_k:]
if re.search(r"[^\da-z]", pre_key, re.I) is not None:
continue
p = dict()
v = self.redis_man.get(item)
p["value"] = v
vs = v.split("_", 2)
if len(vs) < 2:
continue
p["heartbeat_value"] = vs[0]
p["clock_time"] = vs[1]
try:
p["clock_time"] = int(p["clock_time"])
x = time.localtime(p["clock_time"])
p["clock_time2"] = time.strftime("%Y-%m-%d %H:%M:%S", x)
except ValueError:
pass
if len(vs) > 2:
p["current_task"] = vs[2]
p["working"] = True
else:
p["working"] = False
d_wd[pre_key] = p
return d_wd
def task_log(self, *args, **kwargs):
if self.current_task is None or self.current_task.log_path is None:
return
msg = StringTool.join(args, " ")
level = kwargs.pop("level", "INFO")
level = str(level).upper()
if level not in ["INFO", "DEBUG"]:
p_msg_a = [self.current_task.task_key]
if self.current_task.task_sub_key is not None:
p_msg_a.extend([" ", self.current_task.task_sub_key])
p_msg = StringTool.join([p_msg_a, "\n", msg], "")
self.publish_message(p_msg)
if self.upload_log_tag is not None:
upload_info = dict(log_path=self.current_task.log_path,
timestamp=int(time.time()))
self.push_task(StringTool.join_decode(
[self.current_task.task_key, self.work_tag], join_str="_"),
upload_info,
work_tag=self.upload_log_tag)
log_file = self.current_task.log_path
now_time = datetime.now().strftime(TIME_FORMAT)
write_a = ["[", self.heartbeat_value]
if self.worker_index is not None:
write_a.extend([":", self.worker_index])
if self.current_task.task_sub_key is not None:
write_a.extend(["][", self.current_task.task_sub_key])
write_a.extend(["] ", now_time, ": ", level, " ", msg, "\n"])
with open(log_file, "ab", 0) as wl:
u = StringTool.join(write_a, join_str="")
s = StringTool.encode(u)
wl.write(s)
if self.redirect_stdout is False and self.debug is True:
try:
logging.info(s)
except Exception as e:
pass
def read_task_log(self,
work_tag,
key,
sub_key=None,
sub_key_prefix=None,
level="INFO",
max_length=1000000):
"""
:param work_tag:
:param key:
:param sub_key: 为None时查询所有有子key和无子key的日志，为空字符串时仅查询无子key的日志，为具体某个子key时查询具体子key的日志
:param level: 默认为INFO，允许DEBUG，INFO，WARNING，ERROR。其他值认为是INFO
:return:
"""
name = StringTool.join([work_tag, "_", key, ".log"], "")
log_path = StringTool.path_join(self.log_dir, work_tag.lower(), name)
if os.path.exists(log_path) is False:
log_path = StringTool.path_join(self.log_dir, name)
if os.path.exists(log_path) is False:
return False, None
s_log = os.stat(log_path)
read_seek = s_log.st_size - max_length if max_length < s_log.st_size else 0
# 处理参数
if sub_key is not None:
sub_key = StringTool.encode(sub_key)
if sub_key_prefix is not None:
sub_key_prefix = StringTool.encode(sub_key_prefix)
if StringTool.is_string(level) is False:
level = "INFO"
level = level.upper()
if level not in self.log_level:
level = "INFO"
allow_levels = self.log_level[level]
logs_list = []
last_save = False
with open(log_path, "r") as rl:
rl.seek(read_seek)
c = rl.read()
all_lines = c.split("\n")
for line in all_lines:
rl = self.log_compile.match(line)
if rl is not None:
line_sub_key = rl.groups()[0]
log_time = rl.groups()[1]
if len(line_sub_key) >= 2:
line_sub_key = line_sub_key[1:-1]
line_level = rl.groups()[2]
log_msg = rl.groups()[3]
if sub_key is not None and sub_key != line_sub_key:
last_save = False
continue
if sub_key_prefix is not None and line_sub_key.startswith(
sub_key_prefix) is False:
last_save = False
continue
if line_level not in allow_levels:
last_save = False
continue
last_save = True
logs_list.append(
map(StringTool.decode,
[line_sub_key, log_time, line_level, log_msg]))
elif last_save is True:
logs_list[-1][3] = StringTool.join_decode(
[logs_list[-1][3], line])
return True, logs_list
pbs_template = """#PBS -S /bin/bash
#PBS -m n
#PBS -M <*****@*****.**>
"""
pw_info = os.environ.get("JY_PBS_WORKER_INFO", "pbs_worker.info")
with open(pw_info) as pwr:
c_info = pwr.read()
nc_info = c_info % os.environ
info_dir, info_name = os.path.split(pw_info)
temp_info_name = StringTool.join_encode([".", uuid.uuid4().hex, info_name],
join_str="")
temp_info_path = StringTool.path_join(info_dir, temp_info_name)
with open(temp_info_path, "w") as tiw:
tiw.write(StringTool.encode(nc_info))
pbs_worker_config = ConfigParser.ConfigParser()
pbs_worker_config.read(temp_info_path)
os.remove(temp_info_path)
class PBSAgentWorker(RedisWorker):
expect_params_type = dict
def write_pbs_task(self, work_tag, cmd):
save_name = StringTool.join_decode([
self.current_task.task_key, self.current_task.task_sub_key,
int(time.time()), "pbs"
],
join_str=".")
def __init__(self, file_path, mode="w"):
self.file_path = StringTool.encode(file_path)
self.mode = mode
self._w = None
self._w = open(self.file_path, mode=self.mode)
def write_line(self, s):
self._w.write(StringTool.encode(s) + "\n")
def write(self, s):
self._w.write(StringTool.encode(s))
