def load_module(self, role=None, *args, **kwargs):
"""
加载模块
"""
node_funcs = {}
if role is None:
loop_roles = self.main_conf.node_role.split(',')
else:
loop_roles = [role]
for role in loop_roles:
node_funcs.update({
role: load_module("%s,%s" % (app_abs_path("module/common"), app_abs_path("module/%s" % role)))
})
node_funcs[role].update({
"sys.reload_module": self._reload_module,
"sys.reload_env": self._reload_env,
"sys.get_env": self._get_env,
"sys.copy": self._copy,
"sys.get": self._get,
"sys.job_info": self._job_info,
"sys.exprs": self.exprs,
"sys.rsync_module": self._rsync_module,
"sys.reload_node": self._reload_node,
"sys.ping": self.ping,
"sys.roles": self._get_roles,
"sys.funcs": self._get_func,
"sys.version": self._version,
})
return node_funcs
def load_module(self, role=None, *args, **kwargs):
"""
加载模块
"""
node_funcs = {}
if role is None:
loop_roles = self.main_conf.node_role.split(',')
else:
loop_roles = [role]
for role in loop_roles:
node_funcs.update({
role:
load_module("%s,%s" % (app_abs_path("module/common"),
app_abs_path("module/%s" % role)))
})
node_funcs[role].update({
"sys.reload_module": self._reload_module,
"sys.reload_env": self._reload_env,
"sys.get_env": self._get_env,
"sys.copy": self._copy,
"sys.get": self._get,
"sys.job_info": self._job_info,
"sys.exprs": self.exprs,
"sys.rsync_module": self._rsync_module,
"sys.reload_node": self._reload_node,
"sys.ping": self.ping,
"sys.roles": self._get_roles,
"sys.funcs": self._get_func,
"sys.version": self._version,
})
return node_funcs
def get_job(self, role, node_name, jid):
"""
获取任务
@param role string:角色
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
try:
node_path = os.path.join(self.zookeeper_conf.nodes, role, node_name, "jobs", jid)
data = self.zkconn.get(node_path)[0]
data = msgpack.loads(data)
env = data.get("env")
if env == "aes":
key_str = self.main_conf.token
crypt = Crypt(key_str)
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload["status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload["kwargs"] and "remote_path" in payload["kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(local_path):
local_path = os.path.join(local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(app_abs_path(self.main_conf.cache), fid):
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid))
if check_cache(app_abs_path(self.main_conf.cache), fid):
if not make_dirs(os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(os.path.join(app_abs_path(self.main_conf.cache), fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret = data
except (ZKClientError, KeyboardInterrupt), e:
log.error(e.message)
def get_job(self, job_data):
"""
获取任务
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
key_str = self.main_conf.token
crypt = Crypt(key_str)
try:
rets = self.mq.mget_job(job_data)
for node, data in rets.items():
if data:
env = data.get("env")
if env == "aes":
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload["status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload["kwargs"] and "remote_path" in payload["kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(local_path):
local_path = os.path.join(local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(app_abs_path(self.main_conf.cache), fid):
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid))
if check_cache(app_abs_path(self.main_conf.cache), fid):
if not make_dirs(os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(os.path.join(app_abs_path(self.main_conf.cache), fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret[node] = data
except Exception, e:
log.error(traceback.format_exc())
def _rsync_module(self, *args, **kwargs):
"""
def rsync_module(*args, **kwargs) -> 同步模块
@param args list:支持位置参数，例如sys.rsync_module common_tools.py game_tools.py
@param kwargs dict:支持关键字参数，例如:sys.rsync_module mods=common_tools.py,game_tools.py
@return int:1 if success
"""
copy_pair = kwargs.get("copy_pair", [])
copy_ret = []
copy_mods = []
for role, ifile, dfile in copy_pair:
dfile = os.path.join(app_abs_path(self.main_conf.module), dfile)
if role != kwargs["role"]:
continue
copy_mods.append(dfile)
if not ifile:
log.error("rsync_module [%s] error" % dfile)
continue
copy_ret.append(
self._copy(path_pair="%s,%s" % (ifile, os.path.basename(dfile)), remote_path=dfile,
ret_type="full") == dfile)
if all(copy_ret) and copy_ret:
log.info(
"rsync_module [%s %s] ok" % (kwargs["role"], ','.join([os.path.basename(mod) for mod in copy_mods])))
self.node_funcs.update(self.load_module(kwargs["role"]))
return 1
else:
return 0
def load_env(self, role=None, *args, **kwargs):
"""
加载模块
"""
node_envs = {}
roles = self._get_roles()
if not role:
for r in roles:
node_envs.update({
r: load_env("%s,%s" % (app_abs_path("module/common"), app_abs_path("module/%s" % r)))
})
else:
node_envs.update({
role: load_env("%s,%s" % (app_abs_path("module/common"), app_abs_path("module/%s" % role)))
})
return node_envs
def _rsync_module(self, *args, **kwargs):
"""
def rsync_module(*args, **kwargs) -> 同步模块
@param args list:支持位置参数，例如sys.rsync_module common_tools.py game_tools.py
@param kwargs dict:支持关键字参数，例如:sys.rsync_module mods=common_tools.py,game_tools.py
@return int:1 if success
"""
copy_pair = kwargs.get("copy_pair", [])
copy_ret = []
copy_mods = []
for ifile, dfile in copy_pair:
dfile = os.path.join(app_abs_path(self.main_conf.module), dfile)
copy_mods.append(dfile)
if not ifile:
log.error("rsync_module [%s] error" % dfile)
continue
copy_ret.append(
self._copy(path_pair="%s,%s" %
(ifile, os.path.basename(dfile)),
remote_path=dfile,
ret_type="full") == dfile)
if all(copy_ret) and copy_ret:
log.info("rsync_module [%s] ok" %
(','.join([os.path.basename(mod) for mod in copy_mods])))
self.node_funcs.update(self.load_module())
return 1
else:
return 0
def parse_args(self, args=None, values=None):
options, args = super(ConfParser, self).parse_args(args, values)
self.process_config_dir()
logger.setup_file_logger(
app_abs_path(self.config["swall"]["log_file"]),
self.config["swall"]["log_level"])
return options, args
def _mixin_setup(self):
group = optparse.OptionGroup(self, "Options for conf_dir")
self.add_option_group(group)
group.add_option(
'-c',
'--config_dir',
dest='config_dir',
default=app_abs_path('conf/'),
help='Pass in an alternative configuration dir. Default: %default')
def _copy(self, *args, **kwargs):
"""
def copy(*args, **kwargs) -> 拷贝文件到远程 可以增加一个ret_type=full，支持返回文件名
@param args list:支持位置参数，例如 sys.copy /etc/src.tar.gz /tmp/src.tar.gz ret_type=full
@param kwargs dict:支持关键字参数，例如sys.copy local_path=/etc/src.tar.gz remote_path=/tmp/src.tar.gz ret_type=full
@return int:1 if success else 0
"""
if "path_pair" in kwargs and "remote_path" in kwargs:
fid, file_name = kwargs["path_pair"].split(',')
remote_path = kwargs["remote_path"]
make_path = kwargs.get("make_path", 1)
else:
fid, file_name = args[0].split(',')
remote_path = args[1]
make_path = args[2] if len(args) >= 3 else 1
stat = kwargs.get("stat")
ret_type = kwargs.get("ret_type")
if os.path.isdir(remote_path) or remote_path.endswith('/'):
remote_path = os.path.join(remote_path, file_name)
try:
if int(make_path):
make_dirs(os.path.dirname(remote_path))
else:
if not os.path.exists(os.path.dirname(remote_path)):
return ""
except:
log.info(traceback.format_exc())
#如果cache中没有对应的文件，则先从fs中拷贝过来
if not check_cache(app_abs_path(self.main_conf.cache), fid):
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(
fid, os.path.join(app_abs_path(self.main_conf.cache), fid))
#从cache目录中拷贝文件到目标
ret = cp(os.path.join(app_abs_path(self.main_conf.cache), fid),
remote_path, stat)
if ret_type == "full":
return remote_path if ret else ""
else:
return ret
def _mixin_setup(self):
group = optparse.OptionGroup(
self, "Options for conf_dir"
)
self.add_option_group(group)
group.add_option(
'-c', '--config_dir', dest='config_dir',
default=app_abs_path('conf/'),
help='Pass in an alternative configuration dir. Default: %default'
)
def load_env(self, role=None, *args, **kwargs):
"""
加载模块
"""
node_envs = {}
roles = self._get_roles()
if not role:
for r in roles:
node_envs.update({
r:
load_env("%s,%s" % (app_abs_path("module/common"),
app_abs_path("module/%s" % r)))
})
else:
node_envs.update({
role:
load_env("%s,%s" % (app_abs_path("module/common"),
app_abs_path("module/%s" % role)))
})
return node_envs
def _copy(self, *args, **kwargs):
"""
def copy(*args, **kwargs) -> 拷贝文件到远程 可以增加一个ret_type=full，支持返回文件名
@param args list:支持位置参数，例如 sys.copy /etc/src.tar.gz /tmp/src.tar.gz ret_type=full
@param kwargs dict:支持关键字参数，例如sys.copy local_path=/etc/src.tar.gz remote_path=/tmp/src.tar.gz ret_type=full
@return int:1 if success else 0
"""
if "path_pair" in kwargs and "remote_path" in kwargs:
fid, file_name = kwargs["path_pair"].split(',')
remote_path = kwargs["remote_path"]
make_path = kwargs.get("make_path", 1)
else:
fid, file_name = args[0].split(',')
remote_path = args[1]
make_path = args[2] if len(args) >= 3 else 1
stat = kwargs.get("stat")
ret_type = kwargs.get("ret_type")
if os.path.isdir(remote_path) or remote_path.endswith('/'):
remote_path = os.path.join(remote_path, file_name)
try:
if int(make_path):
make_dirs(os.path.dirname(remote_path))
else:
if not os.path.exists(os.path.dirname(remote_path)):
return ""
except:
log.info(traceback.format_exc())
#如果cache中没有对应的文件，则先从fs中拷贝过来
if not check_cache(app_abs_path(self.main_conf.cache), fid):
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid))
#从cache目录中拷贝文件到目标
ret = cp(os.path.join(app_abs_path(self.main_conf.cache), fid), remote_path, stat)
if ret_type == "full":
return remote_path if ret else ""
else:
return ret
def __init__(self, globs=None, exclude_globs=None, wait_all=False, timeout=30, nthread=None,
conf_dir=DEFAULT_CONF_DIR):
self.config = {}
for f in ('swall', 'fs', 'redis'):
abs_path = app_abs_path(os.path.join(conf_dir, "%s.conf" % f))
self.config[f] = agent_config(abs_path)
self.job = Job(self.config, env="aes")
self.globs = globs if globs else ""
self.exclude_globs = exclude_globs if exclude_globs else ""
self.wait_all = wait_all
self.timeout = timeout
self.nthread = nthread
def _get(self, *args, **kwargs):
"""
def get(*args, **kwargs) -> 从远程获取文件
@param args list:支持位置参数，例如 sys.get /tmp/src.tar.gz /etc/src.tar.gz
@param kwargs dict:支持关键字参数，例如sys.get remote_path=/tmp/src.tar.gz local_path=/etc/src.tar.gz
@return string:local_path
"""
if "local_path" in kwargs and "remote_path" in kwargs:
remote_path = kwargs["remote_path"]
else:
remote_path = args[0]
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
return fscli.upload(remote_path)
def __init__(self, globs=None, exclude_globs=None, role=None, wait_all=False, timeout=30, nthread=None,
conf_dir="/data/swall/conf"):
self.config = {}
for f in ('swall', 'zk', 'fs'):
abs_path = app_abs_path(os.path.join(conf_dir, "%s.conf" % f))
self.config[f] = agent_config(abs_path)
self.job = Job(self.config, env="aes")
self.globs = globs if globs else ""
self.exclude_globs = exclude_globs if exclude_globs else ""
self.role = role
self.wait_all = wait_all
self.timeout = timeout
self.nthread = nthread
def _get(self, *args, **kwargs):
"""
def get(*args, **kwargs) -> 从远程获取文件
@param args list:支持位置参数，例如 sys.get /tmp/src.tar.gz /etc/src.tar.gz
@param kwargs dict:支持关键字参数，例如sys.get remote_path=/tmp/src.tar.gz local_path=/etc/src.tar.gz
@return string:local_path
"""
if "local_path" in kwargs and "remote_path" in kwargs:
remote_path = kwargs["remote_path"]
else:
remote_path = args[0]
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
return fscli.upload(remote_path)
def load_module(self, *args, **kwargs):
"""
加载模块
"""
node_funcs = load_module(app_abs_path("module/"))
node_funcs.update({
"sys.reload_module": self._reload_module,
"sys.reload_env": self._reload_env,
"sys.get_env": self._get_env,
"sys.copy": self._copy,
"sys.get": self._get,
"sys.job_info": self._job_info,
"sys.exprs": self.exprs,
"sys.rsync_module": self._rsync_module,
"sys.ping": self.ping,
"sys.funcs": self._get_func,
"sys.version": self._version,
})
return node_funcs
def load_module(self, *args, **kwargs):
"""
加载模块
"""
node_funcs = load_module(app_abs_path("module/"))
node_funcs.update(
{
"sys.reload_module": self._reload_module,
"sys.reload_env": self._reload_env,
"sys.get_env": self._get_env,
"sys.copy": self._copy,
"sys.get": self._get,
"sys.job_info": self._job_info,
"sys.exprs": self.exprs,
"sys.rsync_module": self._rsync_module,
"sys.ping": self.ping,
"sys.funcs": self._get_func,
"sys.version": self._version,
}
)
return node_funcs
def submit_job(self, cmd, roles, nregex, nexclude=None, args=[], kwargs={}, wait_timeout=0, nthread=-1):
"""
提交任务
@param cmd string:需要执行的命令
@param roles string:需要执行的节点类型，如game、admin、server等，多个用|分隔
@param nregex string:节点匹配正则表达式
@param nexclude string:排除节点正则，会从nregex结果排除掉
@param args list:传给cmd命令的位置参数
@param kwargs dict:传给cmd的位置参数
@param wait_timeout int:等待结果的时间
@param nthread int:单个机器上面执行任务的并发数量
@return dict:{
"retcode": 返回值
"extra_data": 其他信息,
"msg": 提示信息,
}
"""
self._gen_jid()
match_nodes = {}
for role in re.split(r"[|,]", roles):
match = self.keeper.get_nodes_by_regex(role, nregex, nexclude)
if match:
match_nodes[role] = match
if not match_nodes:
log.warn("0 node match for %s [%s]" % (self.jid, cmd))
return {
"retcode": 1,
"extra_data": {},
"msg": "send_job complete,0 node match"
}
if cmd == "sys.copy":
if "help" not in args:
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
if "local_path" in kwargs and "remote_path" in kwargs:
local_path = kwargs["local_path"]
else:
local_path = args[0]
fid = fscli.upload(local_path)
if "local_path" in kwargs and "remote_path" in kwargs:
kwargs["path_pair"] = "%s,%s" % (fid, os.path.basename(local_path))
else:
args[0] = "%s,%s" % (fid, os.path.basename(local_path))
if cmd == "sys.rsync_module":
if not args or args[0] != "help":
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
if "mods" in kwargs:
mods = kwargs["mods"].split(',')
elif args:
mods = args
else:
pass
modules = {}
if "mods" in kwargs or args:
for role in match_nodes:
t_path = [
os.path.join(role, mod) for mod in mods
if os.path.exists(os.path.join(app_abs_path(self.main_conf.module), role, mod))
]
t_path.extend([
os.path.join("common", mod) for mod in mods
if os.path.exists(os.path.join(app_abs_path(self.main_conf.module), "common", mod))
])
modules[role] = t_path
else:
for role in match_nodes:
role_mod_path = os.path.join(app_abs_path(self.main_conf.module), role)
t_path = [
os.path.join(role, mod) for mod in os.listdir(role_mod_path)
if mod.endswith(".py")
]
common_mod_path = os.path.join(app_abs_path(self.main_conf.module), "common")
t_path.extend(
[
os.path.join("common", mod) for mod in os.listdir(common_mod_path)
if mod.endswith(".py")
]
)
modules[role] = t_path
copy_pair = []
for role in modules:
for mod in modules[role]:
mod_path = os.path.join(app_abs_path(self.main_conf.module), mod)
fid = fscli.upload(mod_path)
copy_pair.append((role, fid, mod))
kwargs["copy_pair"] = copy_pair
data = {
"env": self.env,
"payload":
{
"jid": self.jid,
"cmd": cmd,
"args": args,
"kwargs": kwargs,
"status": "READY",
"timeout": self.timeout,
"retry_times": self.retry_times
}
}
if nthread != -1:
data["payload"]["nthread"] = nthread
send_ret = {}
for role in match_nodes:
for node_name in match_nodes[role]:
job_data = deepcopy(data)
send_ret.update({"%s.%s" % (role, node_name): self._send_job(job_data, role, node_name)})
if wait_timeout:
rets = {}
@iTimeout(wait_timeout)
def _return(nodes, job_rets):
while 1:
try:
for r in nodes:
for n in nodes[r]:
job_ret = self.get_job(r, n, self.jid)
i_ret = job_ret["payload"].get("return", "")
if not i_ret:
raise SwallAgentError("wait")
if job_rets.get(r):
job_rets[r].update({n: i_ret})
else:
job_rets[r] = {n: i_ret}
except SwallAgentError:
time.sleep(0.1)
else:
break
try:
_return(match_nodes, rets)
except Timeout, e:
log.error(e)
return {
"retcode": 1,
"extra_data": rets,
"msg": "get result complete!"
}
def submit_job(self,
cmd,
nregex,
nexclude=None,
args=[],
kwargs={},
wait_timeout=0,
nthread=-1):
"""
提交任务
@param cmd string:需要执行的命令
@param nregex string:节点匹配正则表达式
@param nexclude string:排除节点正则，会从nregex结果排除掉
@param args list:传给cmd命令的位置参数
@param kwargs dict:传给cmd的位置参数
@param wait_timeout int:等待结果的时间
@param nthread int:单个机器上面执行任务的并发数量
@return dict:{
"retcode": 返回值
"extra_data": 其他信息,
"msg": 提示信息,
}
"""
self._gen_jid()
match_nodes = []
match = self.keeper.get_nodes_by_regex(nregex, nexclude)
if match:
match_nodes = match
if not match_nodes:
log.warn("0 node match for %s [%s]" % (self.jid, cmd))
return {
"retcode": 1,
"extra_data": {},
"msg": "send_job complete,0 node match"
}
if cmd == "sys.copy":
if "help" not in args:
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
if "local_path" in kwargs and "remote_path" in kwargs:
local_path = kwargs["local_path"]
else:
local_path = args[0]
fid = fscli.upload(local_path)
if "local_path" in kwargs and "remote_path" in kwargs:
kwargs["path_pair"] = "%s,%s" % (
fid, os.path.basename(local_path))
else:
args[0] = "%s,%s" % (fid, os.path.basename(local_path))
if cmd == "sys.rsync_module":
if not args or args[0] != "help":
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
modules = [
mod
for mod in os.listdir(app_abs_path(self.main_conf.module))
if mod.endswith(".py")
]
copy_pair = []
for mod in modules:
mod_path = os.path.join(
app_abs_path(self.main_conf.module), mod)
fid = fscli.upload(mod_path)
copy_pair.append((fid, mod))
kwargs["copy_pair"] = copy_pair
data = {
"env": self.env,
"payload": {
"jid": self.jid,
"cmd": cmd,
"args": args,
"kwargs": kwargs,
"status": "READY",
"timeout": self.timeout,
"retry_times": self.retry_times
}
}
if nthread != -1:
data["payload"]["nthread"] = nthread
node_data = []
for node_name in match_nodes:
job_data = deepcopy(data)
node_data.append((job_data, node_name))
ret = self._send_job(node_data)
send_ret = {n: ret for n in match_nodes}
if wait_timeout:
rets = {}
@iTimeout(wait_timeout)
def _return(nodes, job_rets):
while 1:
job_ret = self.get_job([(n, self.jid) for n in nodes])
for node, ret_ in job_ret.iteritems():
if ret_:
i_ret = ret_["payload"].get("return")
if i_ret is not None:
if job_rets:
job_rets.update({node: i_ret})
else:
job_rets = {node: i_ret}
is_wait = False
for ret_ in job_ret.itervalues():
if not ret_:
is_wait = True
else:
i_ret = ret_["payload"].get("return")
if i_ret is None:
is_wait = True
if is_wait:
continue
else:
break
try:
_return(match_nodes, rets)
except Timeout, e:
log.error(e)
return {
"retcode": 1,
"extra_data": rets,
"msg": "get result complete!"
}
def get_job(self, job_data):
"""
获取任务
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
key_str = self.main_conf.token
crypt = Crypt(key_str)
try:
rets = self.mq.mget_job(job_data)
for node, data in rets.items():
if data:
env = data.get("env")
if env == "aes":
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload[
"status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload[
"kwargs"] and "remote_path" in payload[
"kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(
local_path):
local_path = os.path.join(
local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(
app_abs_path(self.main_conf.cache),
fid):
FsClient = load_fclient(
app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(
fid,
os.path.join(
app_abs_path(self.main_conf.cache),
fid))
if check_cache(
app_abs_path(self.main_conf.cache),
fid):
if not make_dirs(
os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(
os.path.join(
app_abs_path(
self.main_conf.cache),
fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret[node] = data
except Exception, e:
log.error(traceback.format_exc())
def submit_job(self, cmd, nregex, nexclude=None, args=[], kwargs={}, wait_timeout=0, nthread=-1):
"""
提交任务
@param cmd string:需要执行的命令
@param nregex string:节点匹配正则表达式
@param nexclude string:排除节点正则，会从nregex结果排除掉
@param args list:传给cmd命令的位置参数
@param kwargs dict:传给cmd的位置参数
@param wait_timeout int:等待结果的时间
@param nthread int:单个机器上面执行任务的并发数量
@return dict:{
"retcode": 返回值
"extra_data": 其他信息,
"msg": 提示信息,
}
"""
self._gen_jid()
match_nodes = []
match = self.keeper.get_nodes_by_regex(nregex, nexclude)
if match:
match_nodes = match
if not match_nodes:
log.warn("0 node match for %s [%s]" % (self.jid, cmd))
return {
"retcode": 1,
"extra_data": {},
"msg": "send_job complete,0 node match"
}
if cmd == "sys.copy":
if "help" not in args:
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
if "local_path" in kwargs and "remote_path" in kwargs:
local_path = kwargs["local_path"]
else:
local_path = args[0]
fid = fscli.upload(local_path)
if "local_path" in kwargs and "remote_path" in kwargs:
kwargs["path_pair"] = "%s,%s" % (fid, os.path.basename(local_path))
else:
args[0] = "%s,%s" % (fid, os.path.basename(local_path))
if cmd == "sys.rsync_module":
if not args or args[0] != "help":
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
modules = [mod for mod in os.listdir(app_abs_path(self.main_conf.module)) if mod.endswith(".py")]
copy_pair = []
for mod in modules:
mod_path = os.path.join(app_abs_path(self.main_conf.module), mod)
fid = fscli.upload(mod_path)
copy_pair.append((fid, mod))
kwargs["copy_pair"] = copy_pair
data = {
"env": self.env,
"payload":
{
"jid": self.jid,
"cmd": cmd,
"args": args,
"kwargs": kwargs,
"status": "READY",
"timeout": self.timeout,
"retry_times": self.retry_times
}
}
if nthread != -1:
data["payload"]["nthread"] = nthread
node_data = []
for node_name in match_nodes:
job_data = deepcopy(data)
node_data.append((job_data, node_name))
ret = self._send_job(node_data)
send_ret = {n: ret for n in match_nodes}
if wait_timeout:
rets = {}
@iTimeout(wait_timeout)
def _return(nodes, job_rets):
while 1:
job_ret = self.get_job([(n, self.jid) for n in nodes])
for node, ret_ in job_ret.iteritems():
if ret_:
i_ret = ret_["payload"].get("return")
if i_ret is not None:
if job_rets:
job_rets.update({node: i_ret})
else:
job_rets = {node: i_ret}
is_wait = False
for ret_ in job_ret.itervalues():
if not ret_:
is_wait = True
else:
i_ret = ret_["payload"].get("return")
if i_ret is None:
is_wait = True
if is_wait:
continue
else:
break
try:
_return(match_nodes, rets)
except Timeout, e:
log.error(e)
return {
"retcode": 1,
"extra_data": rets,
"msg": "get result complete!"
}
def load_env(self, *args, **kwargs):
"""
加载模块
"""
node_envs = load_env(app_abs_path("module/"))
return node_envs
def get_job(self, role, node_name, jid):
"""
获取任务
@param role string:角色
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
try:
node_path = os.path.join(self.zookeeper_conf.nodes, role,
node_name, "jobs", jid)
data = self.zkconn.get(node_path)[0]
data = msgpack.loads(data)
env = data.get("env")
if env == "aes":
key_str = self.main_conf.token
crypt = Crypt(key_str)
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload[
"status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload[
"kwargs"] and "remote_path" in payload["kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(local_path):
local_path = os.path.join(
local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(app_abs_path(self.main_conf.cache),
fid):
FsClient = load_fclient(app_abs_path(
self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(
fid,
os.path.join(
app_abs_path(self.main_conf.cache), fid))
if check_cache(app_abs_path(self.main_conf.cache),
fid):
if not make_dirs(os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(
os.path.join(
app_abs_path(self.main_conf.cache),
fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret = data
except (ZKClientError, KeyboardInterrupt), e:
log.error(e.message)
def parse_args(self, args=None, values=None):
options, args = super(ConfParser, self).parse_args(args, values)
self.process_config_dir()
logger.setup_file_logger(app_abs_path(self.config["swall"]["log_file"]), self.config["swall"]["log_level"])
return options, args
def submit_job(self,
cmd,
roles,
nregex,
nexclude=None,
args=[],
kwargs={},
wait_timeout=0,
nthread=-1):
"""
提交任务
@param cmd string:需要执行的命令
@param roles string:需要执行的节点类型，如game、admin、server等，多个用|分隔
@param nregex string:节点匹配正则表达式
@param nexclude string:排除节点正则，会从nregex结果排除掉
@param args list:传给cmd命令的位置参数
@param kwargs dict:传给cmd的位置参数
@param wait_timeout int:等待结果的时间
@param nthread int:单个机器上面执行任务的并发数量
@return dict:{
"retcode": 返回值
"extra_data": 其他信息,
"msg": 提示信息,
}
"""
self._gen_jid()
match_nodes = {}
for role in re.split(r"[|,]", roles):
match = self.keeper.get_nodes_by_regex(role, nregex, nexclude)
if match:
match_nodes[role] = match
if not match_nodes:
log.warn("0 node match for %s [%s]" % (self.jid, cmd))
return {
"retcode": 1,
"extra_data": {},
"msg": "send_job complete,0 node match"
}
if cmd == "sys.copy":
if "help" not in args:
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
if "local_path" in kwargs and "remote_path" in kwargs:
local_path = kwargs["local_path"]
else:
local_path = args[0]
fid = fscli.upload(local_path)
if "local_path" in kwargs and "remote_path" in kwargs:
kwargs["path_pair"] = "%s,%s" % (
fid, os.path.basename(local_path))
else:
args[0] = "%s,%s" % (fid, os.path.basename(local_path))
if cmd == "sys.rsync_module":
if not args or args[0] != "help":
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
if "mods" in kwargs:
mods = kwargs["mods"].split(',')
elif args:
mods = args
else:
pass
modules = {}
if "mods" in kwargs or args:
for role in match_nodes:
t_path = [
os.path.join(role, mod) for mod in mods
if os.path.exists(
os.path.join(
app_abs_path(self.main_conf.module), role,
mod))
]
t_path.extend([
os.path.join("common", mod) for mod in mods
if os.path.exists(
os.path.join(
app_abs_path(self.main_conf.module),
"common", mod))
])
modules[role] = t_path
else:
for role in match_nodes:
role_mod_path = os.path.join(
app_abs_path(self.main_conf.module), role)
t_path = [
os.path.join(role, mod)
for mod in os.listdir(role_mod_path)
if mod.endswith(".py")
]
common_mod_path = os.path.join(
app_abs_path(self.main_conf.module), "common")
t_path.extend([
os.path.join("common", mod)
for mod in os.listdir(common_mod_path)
if mod.endswith(".py")
])
modules[role] = t_path
copy_pair = []
for role in modules:
for mod in modules[role]:
mod_path = os.path.join(
app_abs_path(self.main_conf.module), mod)
fid = fscli.upload(mod_path)
copy_pair.append((role, fid, mod))
kwargs["copy_pair"] = copy_pair
data = {
"env": self.env,
"payload": {
"jid": self.jid,
"cmd": cmd,
"args": args,
"kwargs": kwargs,
"status": "READY",
"timeout": self.timeout,
"retry_times": self.retry_times
}
}
if nthread != -1:
data["payload"]["nthread"] = nthread
send_ret = {}
for role in match_nodes:
for node_name in match_nodes[role]:
job_data = deepcopy(data)
send_ret.update({
"%s.%s" % (role, node_name):
self._send_job(job_data, role, node_name)
})
if wait_timeout:
rets = {}
@iTimeout(wait_timeout)
def _return(nodes, job_rets):
while 1:
try:
for r in nodes:
for n in nodes[r]:
job_ret = self.get_job(r, n, self.jid)
i_ret = job_ret["payload"].get("return", "")
if not i_ret:
raise SwallAgentError("wait")
if job_rets.get(r):
job_rets[r].update({n: i_ret})
else:
job_rets[r] = {n: i_ret}
except SwallAgentError:
time.sleep(0.1)
else:
break
try:
_return(match_nodes, rets)
except Timeout, e:
log.error(e)
return {
"retcode": 1,
"extra_data": rets,
"msg": "get result complete!"
}
def load_env(self, *args, **kwargs):
"""
加载模块
"""
node_envs = load_env(app_abs_path("module/"))
return node_envs