def get_job(self, role, node_name, jid):
"""
获取任务
@param role string:角色
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
try:
node_path = os.path.join(self.zookeeper_conf.nodes, role, node_name, "jobs", jid)
data = self.zkconn.get(node_path)[0]
data = msgpack.loads(data)
env = data.get("env")
if env == "aes":
key_str = self.main_conf.token
crypt = Crypt(key_str)
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload["status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload["kwargs"] and "remote_path" in payload["kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(local_path):
local_path = os.path.join(local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(app_abs_path(self.main_conf.cache), fid):
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid))
if check_cache(app_abs_path(self.main_conf.cache), fid):
if not make_dirs(os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(os.path.join(app_abs_path(self.main_conf.cache), fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret = data
except (ZKClientError, KeyboardInterrupt), e:
log.error(e.message)
def _get(self, *args, **kwargs):
"""
def get(*args, **kwargs) -> 从远程获取文件
@param args list:支持位置参数，例如 sys.get /tmp/src.tar.gz /etc/src.tar.gz
@param kwargs dict:支持关键字参数，例如sys.get remote_path=/tmp/src.tar.gz local_path=/etc/src.tar.gz
@return string:local_path
"""
if "local_path" in kwargs and "remote_path" in kwargs:
remote_path = kwargs["remote_path"]
else:
remote_path = args[0]
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
return fscli.upload(remote_path)
def get_job(self, job_data):
"""
获取任务
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
key_str = self.main_conf.token
crypt = Crypt(key_str)
try:
rets = self.mq.mget_job(job_data)
for node, data in rets.items():
if data:
env = data.get("env")
if env == "aes":
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload["status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload["kwargs"] and "remote_path" in payload["kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(local_path):
local_path = os.path.join(local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(app_abs_path(self.main_conf.cache), fid):
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid))
if check_cache(app_abs_path(self.main_conf.cache), fid):
if not make_dirs(os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(os.path.join(app_abs_path(self.main_conf.cache), fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret[node] = data
except Exception, e:
log.error(traceback.format_exc())
def _get(self, *args, **kwargs):
"""
def get(*args, **kwargs) -> 从远程获取文件
@param args list:支持位置参数，例如 sys.get /tmp/src.tar.gz /etc/src.tar.gz
@param kwargs dict:支持关键字参数，例如sys.get remote_path=/tmp/src.tar.gz local_path=/etc/src.tar.gz
@return string:local_path
"""
if "local_path" in kwargs and "remote_path" in kwargs:
remote_path = kwargs["remote_path"]
else:
remote_path = args[0]
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
return fscli.upload(remote_path)
def _copy(self, *args, **kwargs):
"""
def copy(*args, **kwargs) -> 拷贝文件到远程 可以增加一个ret_type=full，支持返回文件名
@param args list:支持位置参数，例如 sys.copy /etc/src.tar.gz /tmp/src.tar.gz ret_type=full
@param kwargs dict:支持关键字参数，例如sys.copy local_path=/etc/src.tar.gz remote_path=/tmp/src.tar.gz ret_type=full
@return int:1 if success else 0
"""
if "path_pair" in kwargs and "remote_path" in kwargs:
fid, file_name = kwargs["path_pair"].split(',')
remote_path = kwargs["remote_path"]
make_path = kwargs.get("make_path", 1)
else:
fid, file_name = args[0].split(',')
remote_path = args[1]
make_path = args[2] if len(args) >= 3 else 1
stat = kwargs.get("stat")
ret_type = kwargs.get("ret_type")
if os.path.isdir(remote_path) or remote_path.endswith('/'):
remote_path = os.path.join(remote_path, file_name)
try:
if int(make_path):
make_dirs(os.path.dirname(remote_path))
else:
if not os.path.exists(os.path.dirname(remote_path)):
return ""
except:
log.info(traceback.format_exc())
#如果cache中没有对应的文件，则先从fs中拷贝过来
if not check_cache(app_abs_path(self.main_conf.cache), fid):
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(
fid, os.path.join(app_abs_path(self.main_conf.cache), fid))
#从cache目录中拷贝文件到目标
ret = cp(os.path.join(app_abs_path(self.main_conf.cache), fid),
remote_path, stat)
if ret_type == "full":
return remote_path if ret else ""
else:
return ret
def _copy(self, *args, **kwargs):
"""
def copy(*args, **kwargs) -> 拷贝文件到远程 可以增加一个ret_type=full，支持返回文件名
@param args list:支持位置参数，例如 sys.copy /etc/src.tar.gz /tmp/src.tar.gz ret_type=full
@param kwargs dict:支持关键字参数，例如sys.copy local_path=/etc/src.tar.gz remote_path=/tmp/src.tar.gz ret_type=full
@return int:1 if success else 0
"""
if "path_pair" in kwargs and "remote_path" in kwargs:
fid, file_name = kwargs["path_pair"].split(',')
remote_path = kwargs["remote_path"]
make_path = kwargs.get("make_path", 1)
else:
fid, file_name = args[0].split(',')
remote_path = args[1]
make_path = args[2] if len(args) >= 3 else 1
stat = kwargs.get("stat")
ret_type = kwargs.get("ret_type")
if os.path.isdir(remote_path) or remote_path.endswith('/'):
remote_path = os.path.join(remote_path, file_name)
try:
if int(make_path):
make_dirs(os.path.dirname(remote_path))
else:
if not os.path.exists(os.path.dirname(remote_path)):
return ""
except:
log.info(traceback.format_exc())
#如果cache中没有对应的文件，则先从fs中拷贝过来
if not check_cache(app_abs_path(self.main_conf.cache), fid):
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid))
#从cache目录中拷贝文件到目标
ret = cp(os.path.join(app_abs_path(self.main_conf.cache), fid), remote_path, stat)
if ret_type == "full":
return remote_path if ret else ""
else:
return ret
dist = os.path.join(self.fs_conf.fs_tmp_dir, fid)
self.sftp.stat(dist)
return 1
except IOError:
return 0
def download(self, fid, to_path):
"""
下载文件
@param filename string:需要下载的文件路径，远程文件
@param to_path string:存放本地的文件路径
@return int:1 if success else 0
"""
try:
dist = os.path.join(self.fs_conf.fs_tmp_dir, fid)
self.sftp.get(dist, to_path)
return 1
except IOError, er:
log.error(er)
return 0
if __name__ == "__main__":
client = load_fclient("~/Documents/works/git/swall/plugins/fservice",
ftype="rsync")
print client
scli = client("~/Documents/works/git/swall/conf/swall.conf")
print scli
print scli.upload("/etc/services")
print scli.download("f9f1d3bc559b817e74c13efc3fd1172fbe170d37",
"/tmp/a.txt")
def submit_job(self, cmd, roles, nregex, nexclude=None, args=[], kwargs={}, wait_timeout=0, nthread=-1):
"""
提交任务
@param cmd string:需要执行的命令
@param roles string:需要执行的节点类型，如game、admin、server等，多个用|分隔
@param nregex string:节点匹配正则表达式
@param nexclude string:排除节点正则，会从nregex结果排除掉
@param args list:传给cmd命令的位置参数
@param kwargs dict:传给cmd的位置参数
@param wait_timeout int:等待结果的时间
@param nthread int:单个机器上面执行任务的并发数量
@return dict:{
"retcode": 返回值
"extra_data": 其他信息,
"msg": 提示信息,
}
"""
self._gen_jid()
match_nodes = {}
for role in re.split(r"[|,]", roles):
match = self.keeper.get_nodes_by_regex(role, nregex, nexclude)
if match:
match_nodes[role] = match
if not match_nodes:
log.warn("0 node match for %s [%s]" % (self.jid, cmd))
return {
"retcode": 1,
"extra_data": {},
"msg": "send_job complete,0 node match"
}
if cmd == "sys.copy":
if "help" not in args:
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
if "local_path" in kwargs and "remote_path" in kwargs:
local_path = kwargs["local_path"]
else:
local_path = args[0]
fid = fscli.upload(local_path)
if "local_path" in kwargs and "remote_path" in kwargs:
kwargs["path_pair"] = "%s,%s" % (fid, os.path.basename(local_path))
else:
args[0] = "%s,%s" % (fid, os.path.basename(local_path))
if cmd == "sys.rsync_module":
if not args or args[0] != "help":
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
if "mods" in kwargs:
mods = kwargs["mods"].split(',')
elif args:
mods = args
else:
pass
modules = {}
if "mods" in kwargs or args:
for role in match_nodes:
t_path = [
os.path.join(role, mod) for mod in mods
if os.path.exists(os.path.join(app_abs_path(self.main_conf.module), role, mod))
]
t_path.extend([
os.path.join("common", mod) for mod in mods
if os.path.exists(os.path.join(app_abs_path(self.main_conf.module), "common", mod))
])
modules[role] = t_path
else:
for role in match_nodes:
role_mod_path = os.path.join(app_abs_path(self.main_conf.module), role)
t_path = [
os.path.join(role, mod) for mod in os.listdir(role_mod_path)
if mod.endswith(".py")
]
common_mod_path = os.path.join(app_abs_path(self.main_conf.module), "common")
t_path.extend(
[
os.path.join("common", mod) for mod in os.listdir(common_mod_path)
if mod.endswith(".py")
]
)
modules[role] = t_path
copy_pair = []
for role in modules:
for mod in modules[role]:
mod_path = os.path.join(app_abs_path(self.main_conf.module), mod)
fid = fscli.upload(mod_path)
copy_pair.append((role, fid, mod))
kwargs["copy_pair"] = copy_pair
data = {
"env": self.env,
"payload":
{
"jid": self.jid,
"cmd": cmd,
"args": args,
"kwargs": kwargs,
"status": "READY",
"timeout": self.timeout,
"retry_times": self.retry_times
}
}
if nthread != -1:
data["payload"]["nthread"] = nthread
send_ret = {}
for role in match_nodes:
for node_name in match_nodes[role]:
job_data = deepcopy(data)
send_ret.update({"%s.%s" % (role, node_name): self._send_job(job_data, role, node_name)})
if wait_timeout:
rets = {}
@iTimeout(wait_timeout)
def _return(nodes, job_rets):
while 1:
try:
for r in nodes:
for n in nodes[r]:
job_ret = self.get_job(r, n, self.jid)
i_ret = job_ret["payload"].get("return", "")
if not i_ret:
raise SwallAgentError("wait")
if job_rets.get(r):
job_rets[r].update({n: i_ret})
else:
job_rets[r] = {n: i_ret}
except SwallAgentError:
time.sleep(0.1)
else:
break
try:
_return(match_nodes, rets)
except Timeout, e:
log.error(e)
return {
"retcode": 1,
"extra_data": rets,
"msg": "get result complete!"
}
index = 1
while index <= int(max_retry):
ret = call(
"RSYNC_PASSWORD=%s rsync -a --port=%s --partial %s@%s::swall_fs/%s %s" %
(
self.fs_conf.fs_pass,
self.fs_conf.fs_port,
self.fs_conf.fs_user,
self.fs_conf.fs_host,
fid,
to_path
),
shell=True
)
if ret != 0:
time.sleep(5)
else:
break
index += 1
if index <= int(max_retry):
return 1
else:
return 0
if __name__ == "__main__":
client = load_fclient("~/Documents/works/git/swall/plugins/fservice", ftype="rsync")
print client
scli = client("~/Documents/works/git/swall/conf/swall.conf")
print scli
print scli.upload("/etc/services")
print scli.download("f9f1d3bc559b817e74c13efc3fd1172fbe170d37","/tmp/a.txt")
def submit_job(self,
cmd,
nregex,
nexclude=None,
args=[],
kwargs={},
wait_timeout=0,
nthread=-1):
"""
提交任务
@param cmd string:需要执行的命令
@param nregex string:节点匹配正则表达式
@param nexclude string:排除节点正则，会从nregex结果排除掉
@param args list:传给cmd命令的位置参数
@param kwargs dict:传给cmd的位置参数
@param wait_timeout int:等待结果的时间
@param nthread int:单个机器上面执行任务的并发数量
@return dict:{
"retcode": 返回值
"extra_data": 其他信息,
"msg": 提示信息,
}
"""
self._gen_jid()
match_nodes = []
match = self.keeper.get_nodes_by_regex(nregex, nexclude)
if match:
match_nodes = match
if not match_nodes:
log.warn("0 node match for %s [%s]" % (self.jid, cmd))
return {
"retcode": 1,
"extra_data": {},
"msg": "send_job complete,0 node match"
}
if cmd == "sys.copy":
if "help" not in args:
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
if "local_path" in kwargs and "remote_path" in kwargs:
local_path = kwargs["local_path"]
else:
local_path = args[0]
fid = fscli.upload(local_path)
if "local_path" in kwargs and "remote_path" in kwargs:
kwargs["path_pair"] = "%s,%s" % (
fid, os.path.basename(local_path))
else:
args[0] = "%s,%s" % (fid, os.path.basename(local_path))
if cmd == "sys.rsync_module":
if not args or args[0] != "help":
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
modules = [
mod
for mod in os.listdir(app_abs_path(self.main_conf.module))
if mod.endswith(".py")
]
copy_pair = []
for mod in modules:
mod_path = os.path.join(
app_abs_path(self.main_conf.module), mod)
fid = fscli.upload(mod_path)
copy_pair.append((fid, mod))
kwargs["copy_pair"] = copy_pair
data = {
"env": self.env,
"payload": {
"jid": self.jid,
"cmd": cmd,
"args": args,
"kwargs": kwargs,
"status": "READY",
"timeout": self.timeout,
"retry_times": self.retry_times
}
}
if nthread != -1:
data["payload"]["nthread"] = nthread
node_data = []
for node_name in match_nodes:
job_data = deepcopy(data)
node_data.append((job_data, node_name))
ret = self._send_job(node_data)
send_ret = {n: ret for n in match_nodes}
if wait_timeout:
rets = {}
@iTimeout(wait_timeout)
def _return(nodes, job_rets):
while 1:
job_ret = self.get_job([(n, self.jid) for n in nodes])
for node, ret_ in job_ret.iteritems():
if ret_:
i_ret = ret_["payload"].get("return")
if i_ret is not None:
if job_rets:
job_rets.update({node: i_ret})
else:
job_rets = {node: i_ret}
is_wait = False
for ret_ in job_ret.itervalues():
if not ret_:
is_wait = True
else:
i_ret = ret_["payload"].get("return")
if i_ret is None:
is_wait = True
if is_wait:
continue
else:
break
try:
_return(match_nodes, rets)
except Timeout, e:
log.error(e)
return {
"retcode": 1,
"extra_data": rets,
"msg": "get result complete!"
}
def get_job(self, job_data):
"""
获取任务
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
key_str = self.main_conf.token
crypt = Crypt(key_str)
try:
rets = self.mq.mget_job(job_data)
for node, data in rets.items():
if data:
env = data.get("env")
if env == "aes":
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload[
"status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload[
"kwargs"] and "remote_path" in payload[
"kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(
local_path):
local_path = os.path.join(
local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(
app_abs_path(self.main_conf.cache),
fid):
FsClient = load_fclient(
app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(
fid,
os.path.join(
app_abs_path(self.main_conf.cache),
fid))
if check_cache(
app_abs_path(self.main_conf.cache),
fid):
if not make_dirs(
os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(
os.path.join(
app_abs_path(
self.main_conf.cache),
fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret[node] = data
except Exception, e:
log.error(traceback.format_exc())
def submit_job(self, cmd, nregex, nexclude=None, args=[], kwargs={}, wait_timeout=0, nthread=-1):
"""
提交任务
@param cmd string:需要执行的命令
@param nregex string:节点匹配正则表达式
@param nexclude string:排除节点正则，会从nregex结果排除掉
@param args list:传给cmd命令的位置参数
@param kwargs dict:传给cmd的位置参数
@param wait_timeout int:等待结果的时间
@param nthread int:单个机器上面执行任务的并发数量
@return dict:{
"retcode": 返回值
"extra_data": 其他信息,
"msg": 提示信息,
}
"""
self._gen_jid()
match_nodes = []
match = self.keeper.get_nodes_by_regex(nregex, nexclude)
if match:
match_nodes = match
if not match_nodes:
log.warn("0 node match for %s [%s]" % (self.jid, cmd))
return {
"retcode": 1,
"extra_data": {},
"msg": "send_job complete,0 node match"
}
if cmd == "sys.copy":
if "help" not in args:
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
if "local_path" in kwargs and "remote_path" in kwargs:
local_path = kwargs["local_path"]
else:
local_path = args[0]
fid = fscli.upload(local_path)
if "local_path" in kwargs and "remote_path" in kwargs:
kwargs["path_pair"] = "%s,%s" % (fid, os.path.basename(local_path))
else:
args[0] = "%s,%s" % (fid, os.path.basename(local_path))
if cmd == "sys.rsync_module":
if not args or args[0] != "help":
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
modules = [mod for mod in os.listdir(app_abs_path(self.main_conf.module)) if mod.endswith(".py")]
copy_pair = []
for mod in modules:
mod_path = os.path.join(app_abs_path(self.main_conf.module), mod)
fid = fscli.upload(mod_path)
copy_pair.append((fid, mod))
kwargs["copy_pair"] = copy_pair
data = {
"env": self.env,
"payload":
{
"jid": self.jid,
"cmd": cmd,
"args": args,
"kwargs": kwargs,
"status": "READY",
"timeout": self.timeout,
"retry_times": self.retry_times
}
}
if nthread != -1:
data["payload"]["nthread"] = nthread
node_data = []
for node_name in match_nodes:
job_data = deepcopy(data)
node_data.append((job_data, node_name))
ret = self._send_job(node_data)
send_ret = {n: ret for n in match_nodes}
if wait_timeout:
rets = {}
@iTimeout(wait_timeout)
def _return(nodes, job_rets):
while 1:
job_ret = self.get_job([(n, self.jid) for n in nodes])
for node, ret_ in job_ret.iteritems():
if ret_:
i_ret = ret_["payload"].get("return")
if i_ret is not None:
if job_rets:
job_rets.update({node: i_ret})
else:
job_rets = {node: i_ret}
is_wait = False
for ret_ in job_ret.itervalues():
if not ret_:
is_wait = True
else:
i_ret = ret_["payload"].get("return")
if i_ret is None:
is_wait = True
if is_wait:
continue
else:
break
try:
_return(match_nodes, rets)
except Timeout, e:
log.error(e)
return {
"retcode": 1,
"extra_data": rets,
"msg": "get result complete!"
}
def submit_job(self,
cmd,
roles,
nregex,
nexclude=None,
args=[],
kwargs={},
wait_timeout=0,
nthread=-1):
"""
提交任务
@param cmd string:需要执行的命令
@param roles string:需要执行的节点类型，如game、admin、server等，多个用|分隔
@param nregex string:节点匹配正则表达式
@param nexclude string:排除节点正则，会从nregex结果排除掉
@param args list:传给cmd命令的位置参数
@param kwargs dict:传给cmd的位置参数
@param wait_timeout int:等待结果的时间
@param nthread int:单个机器上面执行任务的并发数量
@return dict:{
"retcode": 返回值
"extra_data": 其他信息,
"msg": 提示信息,
}
"""
self._gen_jid()
match_nodes = {}
for role in re.split(r"[|,]", roles):
match = self.keeper.get_nodes_by_regex(role, nregex, nexclude)
if match:
match_nodes[role] = match
if not match_nodes:
log.warn("0 node match for %s [%s]" % (self.jid, cmd))
return {
"retcode": 1,
"extra_data": {},
"msg": "send_job complete,0 node match"
}
if cmd == "sys.copy":
if "help" not in args:
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
if "local_path" in kwargs and "remote_path" in kwargs:
local_path = kwargs["local_path"]
else:
local_path = args[0]
fid = fscli.upload(local_path)
if "local_path" in kwargs and "remote_path" in kwargs:
kwargs["path_pair"] = "%s,%s" % (
fid, os.path.basename(local_path))
else:
args[0] = "%s,%s" % (fid, os.path.basename(local_path))
if cmd == "sys.rsync_module":
if not args or args[0] != "help":
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
if "mods" in kwargs:
mods = kwargs["mods"].split(',')
elif args:
mods = args
else:
pass
modules = {}
if "mods" in kwargs or args:
for role in match_nodes:
t_path = [
os.path.join(role, mod) for mod in mods
if os.path.exists(
os.path.join(
app_abs_path(self.main_conf.module), role,
mod))
]
t_path.extend([
os.path.join("common", mod) for mod in mods
if os.path.exists(
os.path.join(
app_abs_path(self.main_conf.module),
"common", mod))
])
modules[role] = t_path
else:
for role in match_nodes:
role_mod_path = os.path.join(
app_abs_path(self.main_conf.module), role)
t_path = [
os.path.join(role, mod)
for mod in os.listdir(role_mod_path)
if mod.endswith(".py")
]
common_mod_path = os.path.join(
app_abs_path(self.main_conf.module), "common")
t_path.extend([
os.path.join("common", mod)
for mod in os.listdir(common_mod_path)
if mod.endswith(".py")
])
modules[role] = t_path
copy_pair = []
for role in modules:
for mod in modules[role]:
mod_path = os.path.join(
app_abs_path(self.main_conf.module), mod)
fid = fscli.upload(mod_path)
copy_pair.append((role, fid, mod))
kwargs["copy_pair"] = copy_pair
data = {
"env": self.env,
"payload": {
"jid": self.jid,
"cmd": cmd,
"args": args,
"kwargs": kwargs,
"status": "READY",
"timeout": self.timeout,
"retry_times": self.retry_times
}
}
if nthread != -1:
data["payload"]["nthread"] = nthread
send_ret = {}
for role in match_nodes:
for node_name in match_nodes[role]:
job_data = deepcopy(data)
send_ret.update({
"%s.%s" % (role, node_name):
self._send_job(job_data, role, node_name)
})
if wait_timeout:
rets = {}
@iTimeout(wait_timeout)
def _return(nodes, job_rets):
while 1:
try:
for r in nodes:
for n in nodes[r]:
job_ret = self.get_job(r, n, self.jid)
i_ret = job_ret["payload"].get("return", "")
if not i_ret:
raise SwallAgentError("wait")
if job_rets.get(r):
job_rets[r].update({n: i_ret})
else:
job_rets[r] = {n: i_ret}
except SwallAgentError:
time.sleep(0.1)
else:
break
try:
_return(match_nodes, rets)
except Timeout, e:
log.error(e)
return {
"retcode": 1,
"extra_data": rets,
"msg": "get result complete!"
}
def get_job(self, role, node_name, jid):
"""
获取任务
@param role string:角色
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
try:
node_path = os.path.join(self.zookeeper_conf.nodes, role,
node_name, "jobs", jid)
data = self.zkconn.get(node_path)[0]
data = msgpack.loads(data)
env = data.get("env")
if env == "aes":
key_str = self.main_conf.token
crypt = Crypt(key_str)
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload[
"status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload[
"kwargs"] and "remote_path" in payload["kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(local_path):
local_path = os.path.join(
local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(app_abs_path(self.main_conf.cache),
fid):
FsClient = load_fclient(app_abs_path(
self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(
fid,
os.path.join(
app_abs_path(self.main_conf.cache), fid))
if check_cache(app_abs_path(self.main_conf.cache),
fid):
if not make_dirs(os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(
os.path.join(
app_abs_path(self.main_conf.cache),
fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret = data
except (ZKClientError, KeyboardInterrupt), e:
log.error(e.message)
