def get_job(self, role, node_name, jid):
"""
获取任务
@param role string:角色
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
try:
node_path = os.path.join(self.zookeeper_conf.nodes, role, node_name, "jobs", jid)
data = self.zkconn.get(node_path)[0]
data = msgpack.loads(data)
env = data.get("env")
if env == "aes":
key_str = self.main_conf.token
crypt = Crypt(key_str)
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload["status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload["kwargs"] and "remote_path" in payload["kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(local_path):
local_path = os.path.join(local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(app_abs_path(self.main_conf.cache), fid):
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid))
if check_cache(app_abs_path(self.main_conf.cache), fid):
if not make_dirs(os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(os.path.join(app_abs_path(self.main_conf.cache), fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret = data
except (ZKClientError, KeyboardInterrupt), e:
log.error(e.message)
def get_job(self, job_data):
"""
获取任务
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
key_str = self.main_conf.token
crypt = Crypt(key_str)
try:
rets = self.mq.mget_job(job_data)
for node, data in rets.items():
if data:
env = data.get("env")
if env == "aes":
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload["status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload["kwargs"] and "remote_path" in payload["kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(local_path):
local_path = os.path.join(local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(app_abs_path(self.main_conf.cache), fid):
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid))
if check_cache(app_abs_path(self.main_conf.cache), fid):
if not make_dirs(os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(os.path.join(app_abs_path(self.main_conf.cache), fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret[node] = data
except Exception, e:
log.error(traceback.format_exc())
def _copy(self, *args, **kwargs):
"""
def copy(*args, **kwargs) -> 拷贝文件到远程 可以增加一个ret_type=full，支持返回文件名
@param args list:支持位置参数，例如 sys.copy /etc/src.tar.gz /tmp/src.tar.gz ret_type=full
@param kwargs dict:支持关键字参数，例如sys.copy local_path=/etc/src.tar.gz remote_path=/tmp/src.tar.gz ret_type=full
@return int:1 if success else 0
"""
if "path_pair" in kwargs and "remote_path" in kwargs:
fid, file_name = kwargs["path_pair"].split(',')
remote_path = kwargs["remote_path"]
make_path = kwargs.get("make_path", 1)
else:
fid, file_name = args[0].split(',')
remote_path = args[1]
make_path = args[2] if len(args) >= 3 else 1
stat = kwargs.get("stat")
ret_type = kwargs.get("ret_type")
if os.path.isdir(remote_path) or remote_path.endswith('/'):
remote_path = os.path.join(remote_path, file_name)
try:
if int(make_path):
make_dirs(os.path.dirname(remote_path))
else:
if not os.path.exists(os.path.dirname(remote_path)):
return ""
except:
log.info(traceback.format_exc())
#如果cache中没有对应的文件，则先从fs中拷贝过来
if not check_cache(app_abs_path(self.main_conf.cache), fid):
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(
fid, os.path.join(app_abs_path(self.main_conf.cache), fid))
#从cache目录中拷贝文件到目标
ret = cp(os.path.join(app_abs_path(self.main_conf.cache), fid),
remote_path, stat)
if ret_type == "full":
return remote_path if ret else ""
else:
return ret
def _copy(self, *args, **kwargs):
"""
def copy(*args, **kwargs) -> 拷贝文件到远程 可以增加一个ret_type=full，支持返回文件名
@param args list:支持位置参数，例如 sys.copy /etc/src.tar.gz /tmp/src.tar.gz ret_type=full
@param kwargs dict:支持关键字参数，例如sys.copy local_path=/etc/src.tar.gz remote_path=/tmp/src.tar.gz ret_type=full
@return int:1 if success else 0
"""
if "path_pair" in kwargs and "remote_path" in kwargs:
fid, file_name = kwargs["path_pair"].split(',')
remote_path = kwargs["remote_path"]
make_path = kwargs.get("make_path", 1)
else:
fid, file_name = args[0].split(',')
remote_path = args[1]
make_path = args[2] if len(args) >= 3 else 1
stat = kwargs.get("stat")
ret_type = kwargs.get("ret_type")
if os.path.isdir(remote_path) or remote_path.endswith('/'):
remote_path = os.path.join(remote_path, file_name)
try:
if int(make_path):
make_dirs(os.path.dirname(remote_path))
else:
if not os.path.exists(os.path.dirname(remote_path)):
return ""
except:
log.info(traceback.format_exc())
#如果cache中没有对应的文件，则先从fs中拷贝过来
if not check_cache(app_abs_path(self.main_conf.cache), fid):
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid))
#从cache目录中拷贝文件到目标
ret = cp(os.path.join(app_abs_path(self.main_conf.cache), fid), remote_path, stat)
if ret_type == "full":
return remote_path if ret else ""
else:
return ret
def get_job(self, job_data):
"""
获取任务
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
key_str = self.main_conf.token
crypt = Crypt(key_str)
try:
rets = self.mq.mget_job(job_data)
for node, data in rets.items():
if data:
env = data.get("env")
if env == "aes":
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload[
"status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload[
"kwargs"] and "remote_path" in payload[
"kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(
local_path):
local_path = os.path.join(
local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(
app_abs_path(self.main_conf.cache),
fid):
FsClient = load_fclient(
app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(
fid,
os.path.join(
app_abs_path(self.main_conf.cache),
fid))
if check_cache(
app_abs_path(self.main_conf.cache),
fid):
if not make_dirs(
os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(
os.path.join(
app_abs_path(
self.main_conf.cache),
fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret[node] = data
except Exception, e:
log.error(traceback.format_exc())
def get_job(self, role, node_name, jid):
"""
获取任务
@param role string:角色
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
try:
node_path = os.path.join(self.zookeeper_conf.nodes, role,
node_name, "jobs", jid)
data = self.zkconn.get(node_path)[0]
data = msgpack.loads(data)
env = data.get("env")
if env == "aes":
key_str = self.main_conf.token
crypt = Crypt(key_str)
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload[
"status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload[
"kwargs"] and "remote_path" in payload["kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(local_path):
local_path = os.path.join(
local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(app_abs_path(self.main_conf.cache),
fid):
FsClient = load_fclient(app_abs_path(
self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(
fid,
os.path.join(
app_abs_path(self.main_conf.cache), fid))
if check_cache(app_abs_path(self.main_conf.cache),
fid):
if not make_dirs(os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(
os.path.join(
app_abs_path(self.main_conf.cache),
fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret = data
except (ZKClientError, KeyboardInterrupt), e:
log.error(e.message)
