def getpath(self, k, d=None): """ add in version 1.4.5 """ v = self.get(k, d) v = StringTool.encode(v) if os.path.exists(v) is False: raise WorkerTaskParamsValueTypeError(k, v, "path") return v
def write_example(self, work_tag, params): save_name = StringTool.join_decode([ self.current_task.task_key, self.current_task.task_sub_key, int(time.time()), "json" ], join_str=".") save_dir = StringTool.path_join(example_dir, work_tag) if os.path.isdir(save_dir) is False: os.mkdir(save_dir) save_path = StringTool.path_join(save_dir, save_name) with open(save_path, "w") as wp: wp.write(StringTool.encode(json.dumps(params))) return save_path
def write_pbs_task(self, work_tag, cmd): save_name = StringTool.join_decode([ self.current_task.task_key, self.current_task.task_sub_key, int(time.time()), "pbs" ], join_str=".") save_dir = StringTool.path_join(pbs_task_dir, work_tag) if os.path.isdir(save_dir) is False: os.mkdir(save_dir) save_path = StringTool.path_join(save_dir, save_name) with open(save_path, "w") as wp: cmd = StringTool.join_encode(cmd, join_str=" ") s = StringTool.join_encode([pbs_template, cmd], join_str="\n") wp.write(StringTool.encode(s)) return save_path
def worker_log(self, *args, **kwargs): if self.log_dir is None or is_string(self.log_dir) is False: return msg = StringTool.join(args, " ") level = kwargs.pop("level", "INFO") level = str(level).upper() if level not in ["INFO", "DEBUG"]: self.publish_message(msg) log_file = os.path.join(self.log_dir, "%s.log" % self.work_tag) now_time = datetime.now().strftime(TIME_FORMAT) write_a = ["[", self.heartbeat_value] if self.worker_index is not None: write_a.extend([":", self.worker_index]) write_a.extend(["] ", now_time, ": ", level, " ", msg, "\n"]) with open(log_file, "ab", 0) as wl: u = StringTool.join(write_a, join_str="") s = StringTool.encode(u) wl.write(s) if self.redirect_stdout is False and self.debug is True: try: logging.info(s) except Exception as e: pass
def list_worker_detail(self, work_tag): """ add in version 0.9.7 """ d_wd = dict() work_tag = StringTool.encode(work_tag) key = StringTool.join([self.clock_prefix_key, work_tag, "*"], "_").strip("_") len_k = len(self.clock_prefix_key) + 2 + len(work_tag) ws = self.redis_man.keys(key) for item in ws: if self.redis_man.type(item) != "string": continue pre_key = item[len_k:] if re.search(r"[^\da-z]", pre_key, re.I) is not None: continue p = dict() v = self.redis_man.get(item) p["value"] = v vs = v.split("_", 2) if len(vs) < 2: continue p["heartbeat_value"] = vs[0] p["clock_time"] = vs[1] try: p["clock_time"] = int(p["clock_time"]) x = time.localtime(p["clock_time"]) p["clock_time2"] = time.strftime("%Y-%m-%d %H:%M:%S", x) except ValueError: pass if len(vs) > 2: p["current_task"] = vs[2] p["working"] = True else: p["working"] = False d_wd[pre_key] = p return d_wd
def task_log(self, *args, **kwargs): if self.current_task is None or self.current_task.log_path is None: return msg = StringTool.join(args, " ") level = kwargs.pop("level", "INFO") level = str(level).upper() if level not in ["INFO", "DEBUG"]: p_msg_a = [self.current_task.task_key] if self.current_task.task_sub_key is not None: p_msg_a.extend([" ", self.current_task.task_sub_key]) p_msg = StringTool.join([p_msg_a, "\n", msg], "") self.publish_message(p_msg) if self.upload_log_tag is not None: upload_info = dict(log_path=self.current_task.log_path, timestamp=int(time.time())) self.push_task(StringTool.join_decode( [self.current_task.task_key, self.work_tag], join_str="_"), upload_info, work_tag=self.upload_log_tag) log_file = self.current_task.log_path now_time = datetime.now().strftime(TIME_FORMAT) write_a = ["[", self.heartbeat_value] if self.worker_index is not None: write_a.extend([":", self.worker_index]) if self.current_task.task_sub_key is not None: write_a.extend(["][", self.current_task.task_sub_key]) write_a.extend(["] ", now_time, ": ", level, " ", msg, "\n"]) with open(log_file, "ab", 0) as wl: u = StringTool.join(write_a, join_str="") s = StringTool.encode(u) wl.write(s) if self.redirect_stdout is False and self.debug is True: try: logging.info(s) except Exception as e: pass
def read_task_log(self, work_tag, key, sub_key=None, sub_key_prefix=None, level="INFO", max_length=1000000): """ :param work_tag: :param key: :param sub_key: 为None时查询所有有子key和无子key的日志,为空字符串时仅查询无子key的日志,为具体某个子key时查询具体子key的日志 :param level: 默认为INFO,允许DEBUG,INFO,WARNING,ERROR。其他值认为是INFO :return: """ name = StringTool.join([work_tag, "_", key, ".log"], "") log_path = StringTool.path_join(self.log_dir, work_tag.lower(), name) if os.path.exists(log_path) is False: log_path = StringTool.path_join(self.log_dir, name) if os.path.exists(log_path) is False: return False, None s_log = os.stat(log_path) read_seek = s_log.st_size - max_length if max_length < s_log.st_size else 0 # 处理参数 if sub_key is not None: sub_key = StringTool.encode(sub_key) if sub_key_prefix is not None: sub_key_prefix = StringTool.encode(sub_key_prefix) if StringTool.is_string(level) is False: level = "INFO" level = level.upper() if level not in self.log_level: level = "INFO" allow_levels = self.log_level[level] logs_list = [] last_save = False with open(log_path, "r") as rl: rl.seek(read_seek) c = rl.read() all_lines = c.split("\n") for line in all_lines: rl = self.log_compile.match(line) if rl is not None: line_sub_key = rl.groups()[0] log_time = rl.groups()[1] if len(line_sub_key) >= 2: line_sub_key = line_sub_key[1:-1] line_level = rl.groups()[2] log_msg = rl.groups()[3] if sub_key is not None and sub_key != line_sub_key: last_save = False continue if sub_key_prefix is not None and line_sub_key.startswith( sub_key_prefix) is False: last_save = False continue if line_level not in allow_levels: last_save = False continue last_save = True logs_list.append( map(StringTool.decode, [line_sub_key, log_time, line_level, log_msg])) elif last_save is True: logs_list[-1][3] = StringTool.join_decode( [logs_list[-1][3], line]) return True, logs_list
pbs_template = """#PBS -S /bin/bash #PBS -m n #PBS -M <*****@*****.**> """ pw_info = os.environ.get("JY_PBS_WORKER_INFO", "pbs_worker.info") with open(pw_info) as pwr: c_info = pwr.read() nc_info = c_info % os.environ info_dir, info_name = os.path.split(pw_info) temp_info_name = StringTool.join_encode([".", uuid.uuid4().hex, info_name], join_str="") temp_info_path = StringTool.path_join(info_dir, temp_info_name) with open(temp_info_path, "w") as tiw: tiw.write(StringTool.encode(nc_info)) pbs_worker_config = ConfigParser.ConfigParser() pbs_worker_config.read(temp_info_path) os.remove(temp_info_path) class PBSAgentWorker(RedisWorker): expect_params_type = dict def write_pbs_task(self, work_tag, cmd): save_name = StringTool.join_decode([ self.current_task.task_key, self.current_task.task_sub_key, int(time.time()), "pbs" ], join_str=".")
def __init__(self, file_path, mode="w"): self.file_path = StringTool.encode(file_path) self.mode = mode self._w = None self._w = open(self.file_path, mode=self.mode)
def write_line(self, s): self._w.write(StringTool.encode(s) + "\n")
def write(self, s): self._w.write(StringTool.encode(s))