def write_pbs_task(self, work_tag, cmd):
save_name = StringTool.join_decode([
self.current_task.task_key, self.current_task.task_sub_key,
int(time.time()), "pbs"
],
join_str=".")
save_dir = StringTool.path_join(pbs_task_dir, work_tag)
if os.path.isdir(save_dir) is False:
os.mkdir(save_dir)
save_path = StringTool.path_join(save_dir, save_name)
with open(save_path, "w") as wp:
cmd = StringTool.join_encode(cmd, join_str=" ")
s = StringTool.join_encode([pbs_template, cmd], join_str="\n")
wp.write(StringTool.encode(s))
return save_path
def test(self,
key,
params=None,
params_path=None,
sub_key=None,
report_tag=None,
report_scene=None,
debug=True):
if self.is_running is True: # 一旦运行起来，不可再进入test模式即调用test方法
raise RuntimeError("Can not test, current is running")
self.debug = debug
if params is None and params_path is not None:
with open(params_path, "r") as rp:
c = rp.read()
params = json.loads(c)
task_item = WorkerTask(work_tag=self.work_tag,
task_key=key,
task_sub_key=sub_key,
task_report_tag=report_tag)
if report_scene is not None:
task_item.set(task_report_scene=report_scene)
if self.expect_params_type is not None:
if not isinstance(params, self.expect_params_type):
raise TypeError("params should", self.expect_params_type)
if isinstance(params, dict):
task_item.set(task_params=WorkerTaskParams(**params))
task_item.task_params.debug_func = self.task_debug_log
else:
task_item.set(task_params=params)
if StringTool.is_string(self.log_dir) is True:
log_name = StringTool.join_encode(
[self.work_tag, "_", task_item.task_key, ".log"], join_str="")
task_item.log_path = StringTool.path_join(self.log_dir, log_name)
self.current_task = task_item
return self._execute()
def list_heartbeat_detail(self, work_tag):
"""
add in version 1.0.6
"""
key = StringTool.join_encode(
[self.heartbeat_prefix_key, "_", work_tag])
print(key)
return self.redis_man.get(key)
def delete_heartbeat(self, work_tag):
key = StringTool.join_encode(
[self.heartbeat_prefix_key, "_", work_tag])
return self.redis_man.delete(key)
def parse_task_info(self, task_info):
task_item = WorkerTask(task_info=task_info)
if task_info.startswith("$2") is True:
un_r, data = RedisQueue.unpack_task(task_info)
if un_r is False:
return False, data
if len(data["key"]) <= 0:
return True, None
task_item.set(**data)
if isinstance(task_item.task_params, WorkerTaskParams):
task_item.task_params.debug_func = self.task_debug_log
else:
self.worker_log("handle old data format")
partition_task = task_info.split(",", 3)
if len(partition_task) != 4:
error_msg = "Invalid task %s, task partition length is not 3" % task_info
return False, error_msg
work_tags = partition_task[0].split("|") # 0 work tag 1 return tag
if work_tags[0] != self.work_tag:
error_msg = "Invalid task %s, task not match work tag %s" % (
task_info, self.work_tag)
return False, error_msg
task_item.set(work_tag=work_tags[0])
if len(work_tags) > 1:
task_item.set(task_report_tag=work_tags[1])
keys = partition_task[1].split("|")
if len(keys[0]) <= 0:
return True, None
task_item.set(task_key=keys[0])
if len(keys) > 1:
task_item.set(task_sub_key=keys[1])
if partition_task[2] not in ("string", "json", "report",
"control"):
error_msg = "Invalid task %s, task args type invalid" % task_info
return False, error_msg
params = partition_task[3]
if partition_task[2] in ("json", "report", "control"):
try:
params = json.loads(params)
except ValueError:
error_msg = "Invalid task %s, task args type and args not uniform" % task_info
return False, error_msg
if partition_task[2] == "report":
task_item.set(task_type=TaskType.Report)
task_item.set(task_params=WorkerTask(**params))
elif partition_task[2] == "control":
task_item.set(task_type=TaskType.Control)
if "expected_status" not in params:
return False, "Invalid Task, not found expected_status in params"
expected_status = TaskStatus.parse(params["expected_status"])
if expected_status is None:
return False, "Invalid Task, unknown expected status, %s" % params[
"expected_status"]
task_item.set(task_params=WorkerTaskParams(**params))
task_item.task_params.debug_func = self.task_debug_log
else:
if self.expect_params_type is not None:
if not isinstance(params, self.expect_params_type):
return False, "Invalid task, not expect param type"
if isinstance(self.expect_params_type, dict) is True:
task_item.set(task_params=WorkerTaskParams(**params))
task_item.task_params.debug_func = self.task_debug_log
else:
task_item.set(task_params=params)
if StringTool.is_string(self.log_dir) is True:
log_name = StringTool.join_encode(
[self.work_tag, "_", task_item.task_key, ".log"], join_str="")
task_item.log_path = StringTool.path_join(self.log_dir, log_name)
return True, task_item
os.mkdir(pbs_task_dir)
if os.path.isdir(pbs_log_dir) is False:
os.mkdir(pbs_log_dir)
pbs_template = """#PBS -S /bin/bash
#PBS -m n
#PBS -M <*****@*****.**>
"""
pw_info = os.environ.get("JY_PBS_WORKER_INFO", "pbs_worker.info")
with open(pw_info) as pwr:
c_info = pwr.read()
nc_info = c_info % os.environ
info_dir, info_name = os.path.split(pw_info)
temp_info_name = StringTool.join_encode([".", uuid.uuid4().hex, info_name],
join_str="")
temp_info_path = StringTool.path_join(info_dir, temp_info_name)
with open(temp_info_path, "w") as tiw:
tiw.write(StringTool.encode(nc_info))
pbs_worker_config = ConfigParser.ConfigParser()
pbs_worker_config.read(temp_info_path)
os.remove(temp_info_path)
class PBSAgentWorker(RedisWorker):
expect_params_type = dict
def write_pbs_task(self, work_tag, cmd):
save_name = StringTool.join_decode([
self.current_task.task_key, self.current_task.task_sub_key,
def write_array(self, a, connector="\t"):
s = StringTool.join_encode(a, join_str=connector)
self.write_line(s)