def write_example(self, work_tag, params): save_name = StringTool.join_decode([ self.current_task.task_key, self.current_task.task_sub_key, int(time.time()), "json" ], join_str=".") save_dir = StringTool.path_join(example_dir, work_tag) if os.path.isdir(save_dir) is False: os.mkdir(save_dir) save_path = StringTool.path_join(save_dir, save_name) with open(save_path, "w") as wp: wp.write(StringTool.encode(json.dumps(params))) return save_path
def write_pbs_task(self, work_tag, cmd): save_name = StringTool.join_decode([ self.current_task.task_key, self.current_task.task_sub_key, int(time.time()), "pbs" ], join_str=".") save_dir = StringTool.path_join(pbs_task_dir, work_tag) if os.path.isdir(save_dir) is False: os.mkdir(save_dir) save_path = StringTool.path_join(save_dir, save_name) with open(save_path, "w") as wp: cmd = StringTool.join_encode(cmd, join_str=" ") s = StringTool.join_encode([pbs_template, cmd], join_str="\n") wp.write(StringTool.encode(s)) return save_path
def test(self, key, params=None, params_path=None, sub_key=None, report_tag=None, report_scene=None, debug=True): if self.is_running is True: # 一旦运行起来,不可再进入test模式即调用test方法 raise RuntimeError("Can not test, current is running") self.debug = debug if params is None and params_path is not None: with open(params_path, "r") as rp: c = rp.read() params = json.loads(c) task_item = WorkerTask(work_tag=self.work_tag, task_key=key, task_sub_key=sub_key, task_report_tag=report_tag) if report_scene is not None: task_item.set(task_report_scene=report_scene) if self.expect_params_type is not None: if not isinstance(params, self.expect_params_type): raise TypeError("params should", self.expect_params_type) if isinstance(params, dict): task_item.set(task_params=WorkerTaskParams(**params)) task_item.task_params.debug_func = self.task_debug_log else: task_item.set(task_params=params) if StringTool.is_string(self.log_dir) is True: log_name = StringTool.join_encode( [self.work_tag, "_", task_item.task_key, ".log"], join_str="") task_item.log_path = StringTool.path_join(self.log_dir, log_name) self.current_task = task_item return self._execute()
def read_task_log(self, work_tag, key, sub_key=None, sub_key_prefix=None, level="INFO", max_length=1000000): """ :param work_tag: :param key: :param sub_key: 为None时查询所有有子key和无子key的日志,为空字符串时仅查询无子key的日志,为具体某个子key时查询具体子key的日志 :param level: 默认为INFO,允许DEBUG,INFO,WARNING,ERROR。其他值认为是INFO :return: """ name = StringTool.join([work_tag, "_", key, ".log"], "") log_path = StringTool.path_join(self.log_dir, work_tag.lower(), name) if os.path.exists(log_path) is False: log_path = StringTool.path_join(self.log_dir, name) if os.path.exists(log_path) is False: return False, None s_log = os.stat(log_path) read_seek = s_log.st_size - max_length if max_length < s_log.st_size else 0 # 处理参数 if sub_key is not None: sub_key = StringTool.encode(sub_key) if sub_key_prefix is not None: sub_key_prefix = StringTool.encode(sub_key_prefix) if StringTool.is_string(level) is False: level = "INFO" level = level.upper() if level not in self.log_level: level = "INFO" allow_levels = self.log_level[level] logs_list = [] last_save = False with open(log_path, "r") as rl: rl.seek(read_seek) c = rl.read() all_lines = c.split("\n") for line in all_lines: rl = self.log_compile.match(line) if rl is not None: line_sub_key = rl.groups()[0] log_time = rl.groups()[1] if len(line_sub_key) >= 2: line_sub_key = line_sub_key[1:-1] line_level = rl.groups()[2] log_msg = rl.groups()[3] if sub_key is not None and sub_key != line_sub_key: last_save = False continue if sub_key_prefix is not None and line_sub_key.startswith( sub_key_prefix) is False: last_save = False continue if line_level not in allow_levels: last_save = False continue last_save = True logs_list.append( map(StringTool.decode, [line_sub_key, log_time, line_level, log_msg])) elif last_save is True: logs_list[-1][3] = StringTool.join_decode( [logs_list[-1][3], line]) return True, logs_list
def parse_task_info(self, task_info): task_item = WorkerTask(task_info=task_info) if task_info.startswith("$2") is True: un_r, data = RedisQueue.unpack_task(task_info) if un_r is False: return False, data if len(data["key"]) <= 0: return True, None task_item.set(**data) if isinstance(task_item.task_params, WorkerTaskParams): task_item.task_params.debug_func = self.task_debug_log else: self.worker_log("handle old data format") partition_task = task_info.split(",", 3) if len(partition_task) != 4: error_msg = "Invalid task %s, task partition length is not 3" % task_info return False, error_msg work_tags = partition_task[0].split("|") # 0 work tag 1 return tag if work_tags[0] != self.work_tag: error_msg = "Invalid task %s, task not match work tag %s" % ( task_info, self.work_tag) return False, error_msg task_item.set(work_tag=work_tags[0]) if len(work_tags) > 1: task_item.set(task_report_tag=work_tags[1]) keys = partition_task[1].split("|") if len(keys[0]) <= 0: return True, None task_item.set(task_key=keys[0]) if len(keys) > 1: task_item.set(task_sub_key=keys[1]) if partition_task[2] not in ("string", "json", "report", "control"): error_msg = "Invalid task %s, task args type invalid" % task_info return False, error_msg params = partition_task[3] if partition_task[2] in ("json", "report", "control"): try: params = json.loads(params) except ValueError: error_msg = "Invalid task %s, task args type and args not uniform" % task_info return False, error_msg if partition_task[2] == "report": task_item.set(task_type=TaskType.Report) task_item.set(task_params=WorkerTask(**params)) elif partition_task[2] == "control": task_item.set(task_type=TaskType.Control) if "expected_status" not in params: return False, "Invalid Task, not found expected_status in params" expected_status = TaskStatus.parse(params["expected_status"]) if expected_status is None: return False, "Invalid Task, unknown expected status, %s" % params[ "expected_status"] task_item.set(task_params=WorkerTaskParams(**params)) task_item.task_params.debug_func = self.task_debug_log else: if self.expect_params_type is not None: if not isinstance(params, self.expect_params_type): return False, "Invalid task, not expect param type" if isinstance(self.expect_params_type, dict) is True: task_item.set(task_params=WorkerTaskParams(**params)) task_item.task_params.debug_func = self.task_debug_log else: task_item.set(task_params=params) if StringTool.is_string(self.log_dir) is True: log_name = StringTool.join_encode( [self.work_tag, "_", task_item.task_key, ".log"], join_str="") task_item.log_path = StringTool.path_join(self.log_dir, log_name) return True, task_item
import sys import os import time import json import uuid import tempfile import ConfigParser from JYTools import StringTool from JYTools.JYWorker import RedisWorker, worker_run __author__ = '鹛桑够' sys_tmp_dir = tempfile.gettempdir() log_dir = os.environ.get("JINGD_LOG_DIR", sys_tmp_dir) agent_dir = StringTool.path_join(log_dir, "pbs_agent") if os.path.isdir(agent_dir) is False: os.mkdir(agent_dir) example_dir = StringTool.path_join(agent_dir, "example") pbs_task_dir = StringTool.path_join(agent_dir, "pbs") pbs_log_dir = StringTool.path_join(agent_dir, "log") if os.path.isdir(example_dir) is False: os.mkdir(example_dir) if os.path.isdir(pbs_task_dir) is False: os.mkdir(pbs_task_dir) if os.path.isdir(pbs_log_dir) is False: os.mkdir(pbs_log_dir) pbs_template = """#PBS -S /bin/bash #PBS -m n #PBS -M <*****@*****.**>