def load_module(self, role=None, *args, **kwargs): """ 加载模块 """ node_funcs = {} if role is None: loop_roles = self.main_conf.node_role.split(',') else: loop_roles = [role] for role in loop_roles: node_funcs.update({ role: load_module("%s,%s" % (app_abs_path("module/common"), app_abs_path("module/%s" % role))) }) node_funcs[role].update({ "sys.reload_module": self._reload_module, "sys.reload_env": self._reload_env, "sys.get_env": self._get_env, "sys.copy": self._copy, "sys.get": self._get, "sys.job_info": self._job_info, "sys.exprs": self.exprs, "sys.rsync_module": self._rsync_module, "sys.reload_node": self._reload_node, "sys.ping": self.ping, "sys.roles": self._get_roles, "sys.funcs": self._get_func, "sys.version": self._version, }) return node_funcs
def get_job(self, role, node_name, jid): """ 获取任务 @param role string:角色 @param node_name string:节点名称 @param jid string:任务id @return dict:a job info """ ret = {} try: node_path = os.path.join(self.zookeeper_conf.nodes, role, node_name, "jobs", jid) data = self.zkconn.get(node_path)[0] data = msgpack.loads(data) env = data.get("env") if env == "aes": key_str = self.main_conf.token crypt = Crypt(key_str) data["payload"] = crypt.loads(data.get("payload")) payload = data["payload"] if payload["cmd"] == "sys.get" and payload["status"] == "FINISH" and payload["return"] != "": if payload["args"][0] != "help": fid = payload["return"] if "local_path" in payload["kwargs"] and "remote_path" in payload["kwargs"]: local_path = payload["kwargs"]["local_path"] remote_path = payload["kwargs"]["remote_path"] else: local_path = payload["args"][1] remote_path = payload["args"][0] stat = payload["kwargs"].get("stat") if local_path.endswith('/') or os.path.isdir(local_path): local_path = os.path.join(local_path, os.path.basename(remote_path)) if checksum(local_path) != fid: if not check_cache(app_abs_path(self.main_conf.cache), fid): FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid)) if check_cache(app_abs_path(self.main_conf.cache), fid): if not make_dirs(os.path.dirname(local_path)): log.error("创建目标目录:%s失败" % local_path) if cp(os.path.join(app_abs_path(self.main_conf.cache), fid), local_path, stat): payload["return"] = local_path else: payload["return"] = "" else: payload["return"] = local_path ret = data except (ZKClientError, KeyboardInterrupt), e: log.error(e.message)
def get_job(self, job_data): """ 获取任务 @param node_name string:节点名称 @param jid string:任务id @return dict:a job info """ ret = {} key_str = self.main_conf.token crypt = Crypt(key_str) try: rets = self.mq.mget_job(job_data) for node, data in rets.items(): if data: env = data.get("env") if env == "aes": data["payload"] = crypt.loads(data.get("payload")) payload = data["payload"] if payload["cmd"] == "sys.get" and payload["status"] == "FINISH" and payload["return"] != "": if payload["args"][0] != "help": fid = payload["return"] if "local_path" in payload["kwargs"] and "remote_path" in payload["kwargs"]: local_path = payload["kwargs"]["local_path"] remote_path = payload["kwargs"]["remote_path"] else: local_path = payload["args"][1] remote_path = payload["args"][0] stat = payload["kwargs"].get("stat") if local_path.endswith('/') or os.path.isdir(local_path): local_path = os.path.join(local_path, os.path.basename(remote_path)) if checksum(local_path) != fid: if not check_cache(app_abs_path(self.main_conf.cache), fid): FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid)) if check_cache(app_abs_path(self.main_conf.cache), fid): if not make_dirs(os.path.dirname(local_path)): log.error("创建目标目录:%s失败" % local_path) if cp(os.path.join(app_abs_path(self.main_conf.cache), fid), local_path, stat): payload["return"] = local_path else: payload["return"] = "" else: payload["return"] = local_path ret[node] = data except Exception, e: log.error(traceback.format_exc())
def _rsync_module(self, *args, **kwargs): """ def rsync_module(*args, **kwargs) -> 同步模块 @param args list:支持位置参数,例如sys.rsync_module common_tools.py game_tools.py @param kwargs dict:支持关键字参数,例如:sys.rsync_module mods=common_tools.py,game_tools.py @return int:1 if success """ copy_pair = kwargs.get("copy_pair", []) copy_ret = [] copy_mods = [] for role, ifile, dfile in copy_pair: dfile = os.path.join(app_abs_path(self.main_conf.module), dfile) if role != kwargs["role"]: continue copy_mods.append(dfile) if not ifile: log.error("rsync_module [%s] error" % dfile) continue copy_ret.append( self._copy(path_pair="%s,%s" % (ifile, os.path.basename(dfile)), remote_path=dfile, ret_type="full") == dfile) if all(copy_ret) and copy_ret: log.info( "rsync_module [%s %s] ok" % (kwargs["role"], ','.join([os.path.basename(mod) for mod in copy_mods]))) self.node_funcs.update(self.load_module(kwargs["role"])) return 1 else: return 0
def load_env(self, role=None, *args, **kwargs): """ 加载模块 """ node_envs = {} roles = self._get_roles() if not role: for r in roles: node_envs.update({ r: load_env("%s,%s" % (app_abs_path("module/common"), app_abs_path("module/%s" % r))) }) else: node_envs.update({ role: load_env("%s,%s" % (app_abs_path("module/common"), app_abs_path("module/%s" % role))) }) return node_envs
def _rsync_module(self, *args, **kwargs): """ def rsync_module(*args, **kwargs) -> 同步模块 @param args list:支持位置参数,例如sys.rsync_module common_tools.py game_tools.py @param kwargs dict:支持关键字参数,例如:sys.rsync_module mods=common_tools.py,game_tools.py @return int:1 if success """ copy_pair = kwargs.get("copy_pair", []) copy_ret = [] copy_mods = [] for ifile, dfile in copy_pair: dfile = os.path.join(app_abs_path(self.main_conf.module), dfile) copy_mods.append(dfile) if not ifile: log.error("rsync_module [%s] error" % dfile) continue copy_ret.append( self._copy(path_pair="%s,%s" % (ifile, os.path.basename(dfile)), remote_path=dfile, ret_type="full") == dfile) if all(copy_ret) and copy_ret: log.info("rsync_module [%s] ok" % (','.join([os.path.basename(mod) for mod in copy_mods]))) self.node_funcs.update(self.load_module()) return 1 else: return 0
def parse_args(self, args=None, values=None): options, args = super(ConfParser, self).parse_args(args, values) self.process_config_dir() logger.setup_file_logger( app_abs_path(self.config["swall"]["log_file"]), self.config["swall"]["log_level"]) return options, args
def _mixin_setup(self): group = optparse.OptionGroup(self, "Options for conf_dir") self.add_option_group(group) group.add_option( '-c', '--config_dir', dest='config_dir', default=app_abs_path('conf/'), help='Pass in an alternative configuration dir. Default: %default')
def _copy(self, *args, **kwargs): """ def copy(*args, **kwargs) -> 拷贝文件到远程 可以增加一个ret_type=full,支持返回文件名 @param args list:支持位置参数,例如 sys.copy /etc/src.tar.gz /tmp/src.tar.gz ret_type=full @param kwargs dict:支持关键字参数,例如sys.copy local_path=/etc/src.tar.gz remote_path=/tmp/src.tar.gz ret_type=full @return int:1 if success else 0 """ if "path_pair" in kwargs and "remote_path" in kwargs: fid, file_name = kwargs["path_pair"].split(',') remote_path = kwargs["remote_path"] make_path = kwargs.get("make_path", 1) else: fid, file_name = args[0].split(',') remote_path = args[1] make_path = args[2] if len(args) >= 3 else 1 stat = kwargs.get("stat") ret_type = kwargs.get("ret_type") if os.path.isdir(remote_path) or remote_path.endswith('/'): remote_path = os.path.join(remote_path, file_name) try: if int(make_path): make_dirs(os.path.dirname(remote_path)) else: if not os.path.exists(os.path.dirname(remote_path)): return "" except: log.info(traceback.format_exc()) #如果cache中没有对应的文件,则先从fs中拷贝过来 if not check_cache(app_abs_path(self.main_conf.cache), fid): FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) fscli.download( fid, os.path.join(app_abs_path(self.main_conf.cache), fid)) #从cache目录中拷贝文件到目标 ret = cp(os.path.join(app_abs_path(self.main_conf.cache), fid), remote_path, stat) if ret_type == "full": return remote_path if ret else "" else: return ret
def _mixin_setup(self): group = optparse.OptionGroup( self, "Options for conf_dir" ) self.add_option_group(group) group.add_option( '-c', '--config_dir', dest='config_dir', default=app_abs_path('conf/'), help='Pass in an alternative configuration dir. Default: %default' )
def _copy(self, *args, **kwargs): """ def copy(*args, **kwargs) -> 拷贝文件到远程 可以增加一个ret_type=full,支持返回文件名 @param args list:支持位置参数,例如 sys.copy /etc/src.tar.gz /tmp/src.tar.gz ret_type=full @param kwargs dict:支持关键字参数,例如sys.copy local_path=/etc/src.tar.gz remote_path=/tmp/src.tar.gz ret_type=full @return int:1 if success else 0 """ if "path_pair" in kwargs and "remote_path" in kwargs: fid, file_name = kwargs["path_pair"].split(',') remote_path = kwargs["remote_path"] make_path = kwargs.get("make_path", 1) else: fid, file_name = args[0].split(',') remote_path = args[1] make_path = args[2] if len(args) >= 3 else 1 stat = kwargs.get("stat") ret_type = kwargs.get("ret_type") if os.path.isdir(remote_path) or remote_path.endswith('/'): remote_path = os.path.join(remote_path, file_name) try: if int(make_path): make_dirs(os.path.dirname(remote_path)) else: if not os.path.exists(os.path.dirname(remote_path)): return "" except: log.info(traceback.format_exc()) #如果cache中没有对应的文件,则先从fs中拷贝过来 if not check_cache(app_abs_path(self.main_conf.cache), fid): FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid)) #从cache目录中拷贝文件到目标 ret = cp(os.path.join(app_abs_path(self.main_conf.cache), fid), remote_path, stat) if ret_type == "full": return remote_path if ret else "" else: return ret
def __init__(self, globs=None, exclude_globs=None, wait_all=False, timeout=30, nthread=None, conf_dir=DEFAULT_CONF_DIR): self.config = {} for f in ('swall', 'fs', 'redis'): abs_path = app_abs_path(os.path.join(conf_dir, "%s.conf" % f)) self.config[f] = agent_config(abs_path) self.job = Job(self.config, env="aes") self.globs = globs if globs else "" self.exclude_globs = exclude_globs if exclude_globs else "" self.wait_all = wait_all self.timeout = timeout self.nthread = nthread
def _get(self, *args, **kwargs): """ def get(*args, **kwargs) -> 从远程获取文件 @param args list:支持位置参数,例如 sys.get /tmp/src.tar.gz /etc/src.tar.gz @param kwargs dict:支持关键字参数,例如sys.get remote_path=/tmp/src.tar.gz local_path=/etc/src.tar.gz @return string:local_path """ if "local_path" in kwargs and "remote_path" in kwargs: remote_path = kwargs["remote_path"] else: remote_path = args[0] FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) return fscli.upload(remote_path)
def __init__(self, globs=None, exclude_globs=None, role=None, wait_all=False, timeout=30, nthread=None, conf_dir="/data/swall/conf"): self.config = {} for f in ('swall', 'zk', 'fs'): abs_path = app_abs_path(os.path.join(conf_dir, "%s.conf" % f)) self.config[f] = agent_config(abs_path) self.job = Job(self.config, env="aes") self.globs = globs if globs else "" self.exclude_globs = exclude_globs if exclude_globs else "" self.role = role self.wait_all = wait_all self.timeout = timeout self.nthread = nthread
def load_module(self, *args, **kwargs): """ 加载模块 """ node_funcs = load_module(app_abs_path("module/")) node_funcs.update({ "sys.reload_module": self._reload_module, "sys.reload_env": self._reload_env, "sys.get_env": self._get_env, "sys.copy": self._copy, "sys.get": self._get, "sys.job_info": self._job_info, "sys.exprs": self.exprs, "sys.rsync_module": self._rsync_module, "sys.ping": self.ping, "sys.funcs": self._get_func, "sys.version": self._version, }) return node_funcs
def load_module(self, *args, **kwargs): """ 加载模块 """ node_funcs = load_module(app_abs_path("module/")) node_funcs.update( { "sys.reload_module": self._reload_module, "sys.reload_env": self._reload_env, "sys.get_env": self._get_env, "sys.copy": self._copy, "sys.get": self._get, "sys.job_info": self._job_info, "sys.exprs": self.exprs, "sys.rsync_module": self._rsync_module, "sys.ping": self.ping, "sys.funcs": self._get_func, "sys.version": self._version, } ) return node_funcs
def submit_job(self, cmd, roles, nregex, nexclude=None, args=[], kwargs={}, wait_timeout=0, nthread=-1): """ 提交任务 @param cmd string:需要执行的命令 @param roles string:需要执行的节点类型,如game、admin、server等,多个用|分隔 @param nregex string:节点匹配正则表达式 @param nexclude string:排除节点正则,会从nregex结果排除掉 @param args list:传给cmd命令的位置参数 @param kwargs dict:传给cmd的位置参数 @param wait_timeout int:等待结果的时间 @param nthread int:单个机器上面执行任务的并发数量 @return dict:{ "retcode": 返回值 "extra_data": 其他信息, "msg": 提示信息, } """ self._gen_jid() match_nodes = {} for role in re.split(r"[|,]", roles): match = self.keeper.get_nodes_by_regex(role, nregex, nexclude) if match: match_nodes[role] = match if not match_nodes: log.warn("0 node match for %s [%s]" % (self.jid, cmd)) return { "retcode": 1, "extra_data": {}, "msg": "send_job complete,0 node match" } if cmd == "sys.copy": if "help" not in args: FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) if "local_path" in kwargs and "remote_path" in kwargs: local_path = kwargs["local_path"] else: local_path = args[0] fid = fscli.upload(local_path) if "local_path" in kwargs and "remote_path" in kwargs: kwargs["path_pair"] = "%s,%s" % (fid, os.path.basename(local_path)) else: args[0] = "%s,%s" % (fid, os.path.basename(local_path)) if cmd == "sys.rsync_module": if not args or args[0] != "help": FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) if "mods" in kwargs: mods = kwargs["mods"].split(',') elif args: mods = args else: pass modules = {} if "mods" in kwargs or args: for role in match_nodes: t_path = [ os.path.join(role, mod) for mod in mods if os.path.exists(os.path.join(app_abs_path(self.main_conf.module), role, mod)) ] t_path.extend([ os.path.join("common", mod) for mod in mods if os.path.exists(os.path.join(app_abs_path(self.main_conf.module), "common", mod)) ]) modules[role] = t_path else: for role in match_nodes: role_mod_path = os.path.join(app_abs_path(self.main_conf.module), role) t_path = [ os.path.join(role, mod) for mod in os.listdir(role_mod_path) if mod.endswith(".py") ] common_mod_path = os.path.join(app_abs_path(self.main_conf.module), "common") t_path.extend( [ os.path.join("common", mod) for mod in os.listdir(common_mod_path) if mod.endswith(".py") ] ) modules[role] = t_path copy_pair = [] for role in modules: for mod in modules[role]: mod_path = os.path.join(app_abs_path(self.main_conf.module), mod) fid = fscli.upload(mod_path) copy_pair.append((role, fid, mod)) kwargs["copy_pair"] = copy_pair data = { "env": self.env, "payload": { "jid": self.jid, "cmd": cmd, "args": args, "kwargs": kwargs, "status": "READY", "timeout": self.timeout, "retry_times": self.retry_times } } if nthread != -1: data["payload"]["nthread"] = nthread send_ret = {} for role in match_nodes: for node_name in match_nodes[role]: job_data = deepcopy(data) send_ret.update({"%s.%s" % (role, node_name): self._send_job(job_data, role, node_name)}) if wait_timeout: rets = {} @iTimeout(wait_timeout) def _return(nodes, job_rets): while 1: try: for r in nodes: for n in nodes[r]: job_ret = self.get_job(r, n, self.jid) i_ret = job_ret["payload"].get("return", "") if not i_ret: raise SwallAgentError("wait") if job_rets.get(r): job_rets[r].update({n: i_ret}) else: job_rets[r] = {n: i_ret} except SwallAgentError: time.sleep(0.1) else: break try: _return(match_nodes, rets) except Timeout, e: log.error(e) return { "retcode": 1, "extra_data": rets, "msg": "get result complete!" }
def submit_job(self, cmd, nregex, nexclude=None, args=[], kwargs={}, wait_timeout=0, nthread=-1): """ 提交任务 @param cmd string:需要执行的命令 @param nregex string:节点匹配正则表达式 @param nexclude string:排除节点正则,会从nregex结果排除掉 @param args list:传给cmd命令的位置参数 @param kwargs dict:传给cmd的位置参数 @param wait_timeout int:等待结果的时间 @param nthread int:单个机器上面执行任务的并发数量 @return dict:{ "retcode": 返回值 "extra_data": 其他信息, "msg": 提示信息, } """ self._gen_jid() match_nodes = [] match = self.keeper.get_nodes_by_regex(nregex, nexclude) if match: match_nodes = match if not match_nodes: log.warn("0 node match for %s [%s]" % (self.jid, cmd)) return { "retcode": 1, "extra_data": {}, "msg": "send_job complete,0 node match" } if cmd == "sys.copy": if "help" not in args: FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) if "local_path" in kwargs and "remote_path" in kwargs: local_path = kwargs["local_path"] else: local_path = args[0] fid = fscli.upload(local_path) if "local_path" in kwargs and "remote_path" in kwargs: kwargs["path_pair"] = "%s,%s" % ( fid, os.path.basename(local_path)) else: args[0] = "%s,%s" % (fid, os.path.basename(local_path)) if cmd == "sys.rsync_module": if not args or args[0] != "help": FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) modules = [ mod for mod in os.listdir(app_abs_path(self.main_conf.module)) if mod.endswith(".py") ] copy_pair = [] for mod in modules: mod_path = os.path.join( app_abs_path(self.main_conf.module), mod) fid = fscli.upload(mod_path) copy_pair.append((fid, mod)) kwargs["copy_pair"] = copy_pair data = { "env": self.env, "payload": { "jid": self.jid, "cmd": cmd, "args": args, "kwargs": kwargs, "status": "READY", "timeout": self.timeout, "retry_times": self.retry_times } } if nthread != -1: data["payload"]["nthread"] = nthread node_data = [] for node_name in match_nodes: job_data = deepcopy(data) node_data.append((job_data, node_name)) ret = self._send_job(node_data) send_ret = {n: ret for n in match_nodes} if wait_timeout: rets = {} @iTimeout(wait_timeout) def _return(nodes, job_rets): while 1: job_ret = self.get_job([(n, self.jid) for n in nodes]) for node, ret_ in job_ret.iteritems(): if ret_: i_ret = ret_["payload"].get("return") if i_ret is not None: if job_rets: job_rets.update({node: i_ret}) else: job_rets = {node: i_ret} is_wait = False for ret_ in job_ret.itervalues(): if not ret_: is_wait = True else: i_ret = ret_["payload"].get("return") if i_ret is None: is_wait = True if is_wait: continue else: break try: _return(match_nodes, rets) except Timeout, e: log.error(e) return { "retcode": 1, "extra_data": rets, "msg": "get result complete!" }
def get_job(self, job_data): """ 获取任务 @param node_name string:节点名称 @param jid string:任务id @return dict:a job info """ ret = {} key_str = self.main_conf.token crypt = Crypt(key_str) try: rets = self.mq.mget_job(job_data) for node, data in rets.items(): if data: env = data.get("env") if env == "aes": data["payload"] = crypt.loads(data.get("payload")) payload = data["payload"] if payload["cmd"] == "sys.get" and payload[ "status"] == "FINISH" and payload["return"] != "": if payload["args"][0] != "help": fid = payload["return"] if "local_path" in payload[ "kwargs"] and "remote_path" in payload[ "kwargs"]: local_path = payload["kwargs"]["local_path"] remote_path = payload["kwargs"]["remote_path"] else: local_path = payload["args"][1] remote_path = payload["args"][0] stat = payload["kwargs"].get("stat") if local_path.endswith('/') or os.path.isdir( local_path): local_path = os.path.join( local_path, os.path.basename(remote_path)) if checksum(local_path) != fid: if not check_cache( app_abs_path(self.main_conf.cache), fid): FsClient = load_fclient( app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) fscli.download( fid, os.path.join( app_abs_path(self.main_conf.cache), fid)) if check_cache( app_abs_path(self.main_conf.cache), fid): if not make_dirs( os.path.dirname(local_path)): log.error("创建目标目录:%s失败" % local_path) if cp( os.path.join( app_abs_path( self.main_conf.cache), fid), local_path, stat): payload["return"] = local_path else: payload["return"] = "" else: payload["return"] = local_path ret[node] = data except Exception, e: log.error(traceback.format_exc())
def submit_job(self, cmd, nregex, nexclude=None, args=[], kwargs={}, wait_timeout=0, nthread=-1): """ 提交任务 @param cmd string:需要执行的命令 @param nregex string:节点匹配正则表达式 @param nexclude string:排除节点正则,会从nregex结果排除掉 @param args list:传给cmd命令的位置参数 @param kwargs dict:传给cmd的位置参数 @param wait_timeout int:等待结果的时间 @param nthread int:单个机器上面执行任务的并发数量 @return dict:{ "retcode": 返回值 "extra_data": 其他信息, "msg": 提示信息, } """ self._gen_jid() match_nodes = [] match = self.keeper.get_nodes_by_regex(nregex, nexclude) if match: match_nodes = match if not match_nodes: log.warn("0 node match for %s [%s]" % (self.jid, cmd)) return { "retcode": 1, "extra_data": {}, "msg": "send_job complete,0 node match" } if cmd == "sys.copy": if "help" not in args: FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) if "local_path" in kwargs and "remote_path" in kwargs: local_path = kwargs["local_path"] else: local_path = args[0] fid = fscli.upload(local_path) if "local_path" in kwargs and "remote_path" in kwargs: kwargs["path_pair"] = "%s,%s" % (fid, os.path.basename(local_path)) else: args[0] = "%s,%s" % (fid, os.path.basename(local_path)) if cmd == "sys.rsync_module": if not args or args[0] != "help": FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) modules = [mod for mod in os.listdir(app_abs_path(self.main_conf.module)) if mod.endswith(".py")] copy_pair = [] for mod in modules: mod_path = os.path.join(app_abs_path(self.main_conf.module), mod) fid = fscli.upload(mod_path) copy_pair.append((fid, mod)) kwargs["copy_pair"] = copy_pair data = { "env": self.env, "payload": { "jid": self.jid, "cmd": cmd, "args": args, "kwargs": kwargs, "status": "READY", "timeout": self.timeout, "retry_times": self.retry_times } } if nthread != -1: data["payload"]["nthread"] = nthread node_data = [] for node_name in match_nodes: job_data = deepcopy(data) node_data.append((job_data, node_name)) ret = self._send_job(node_data) send_ret = {n: ret for n in match_nodes} if wait_timeout: rets = {} @iTimeout(wait_timeout) def _return(nodes, job_rets): while 1: job_ret = self.get_job([(n, self.jid) for n in nodes]) for node, ret_ in job_ret.iteritems(): if ret_: i_ret = ret_["payload"].get("return") if i_ret is not None: if job_rets: job_rets.update({node: i_ret}) else: job_rets = {node: i_ret} is_wait = False for ret_ in job_ret.itervalues(): if not ret_: is_wait = True else: i_ret = ret_["payload"].get("return") if i_ret is None: is_wait = True if is_wait: continue else: break try: _return(match_nodes, rets) except Timeout, e: log.error(e) return { "retcode": 1, "extra_data": rets, "msg": "get result complete!" }
def load_env(self, *args, **kwargs): """ 加载模块 """ node_envs = load_env(app_abs_path("module/")) return node_envs
def get_job(self, role, node_name, jid): """ 获取任务 @param role string:角色 @param node_name string:节点名称 @param jid string:任务id @return dict:a job info """ ret = {} try: node_path = os.path.join(self.zookeeper_conf.nodes, role, node_name, "jobs", jid) data = self.zkconn.get(node_path)[0] data = msgpack.loads(data) env = data.get("env") if env == "aes": key_str = self.main_conf.token crypt = Crypt(key_str) data["payload"] = crypt.loads(data.get("payload")) payload = data["payload"] if payload["cmd"] == "sys.get" and payload[ "status"] == "FINISH" and payload["return"] != "": if payload["args"][0] != "help": fid = payload["return"] if "local_path" in payload[ "kwargs"] and "remote_path" in payload["kwargs"]: local_path = payload["kwargs"]["local_path"] remote_path = payload["kwargs"]["remote_path"] else: local_path = payload["args"][1] remote_path = payload["args"][0] stat = payload["kwargs"].get("stat") if local_path.endswith('/') or os.path.isdir(local_path): local_path = os.path.join( local_path, os.path.basename(remote_path)) if checksum(local_path) != fid: if not check_cache(app_abs_path(self.main_conf.cache), fid): FsClient = load_fclient(app_abs_path( self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) fscli.download( fid, os.path.join( app_abs_path(self.main_conf.cache), fid)) if check_cache(app_abs_path(self.main_conf.cache), fid): if not make_dirs(os.path.dirname(local_path)): log.error("创建目标目录:%s失败" % local_path) if cp( os.path.join( app_abs_path(self.main_conf.cache), fid), local_path, stat): payload["return"] = local_path else: payload["return"] = "" else: payload["return"] = local_path ret = data except (ZKClientError, KeyboardInterrupt), e: log.error(e.message)
def parse_args(self, args=None, values=None): options, args = super(ConfParser, self).parse_args(args, values) self.process_config_dir() logger.setup_file_logger(app_abs_path(self.config["swall"]["log_file"]), self.config["swall"]["log_level"]) return options, args
def submit_job(self, cmd, roles, nregex, nexclude=None, args=[], kwargs={}, wait_timeout=0, nthread=-1): """ 提交任务 @param cmd string:需要执行的命令 @param roles string:需要执行的节点类型,如game、admin、server等,多个用|分隔 @param nregex string:节点匹配正则表达式 @param nexclude string:排除节点正则,会从nregex结果排除掉 @param args list:传给cmd命令的位置参数 @param kwargs dict:传给cmd的位置参数 @param wait_timeout int:等待结果的时间 @param nthread int:单个机器上面执行任务的并发数量 @return dict:{ "retcode": 返回值 "extra_data": 其他信息, "msg": 提示信息, } """ self._gen_jid() match_nodes = {} for role in re.split(r"[|,]", roles): match = self.keeper.get_nodes_by_regex(role, nregex, nexclude) if match: match_nodes[role] = match if not match_nodes: log.warn("0 node match for %s [%s]" % (self.jid, cmd)) return { "retcode": 1, "extra_data": {}, "msg": "send_job complete,0 node match" } if cmd == "sys.copy": if "help" not in args: FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) if "local_path" in kwargs and "remote_path" in kwargs: local_path = kwargs["local_path"] else: local_path = args[0] fid = fscli.upload(local_path) if "local_path" in kwargs and "remote_path" in kwargs: kwargs["path_pair"] = "%s,%s" % ( fid, os.path.basename(local_path)) else: args[0] = "%s,%s" % (fid, os.path.basename(local_path)) if cmd == "sys.rsync_module": if not args or args[0] != "help": FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) if "mods" in kwargs: mods = kwargs["mods"].split(',') elif args: mods = args else: pass modules = {} if "mods" in kwargs or args: for role in match_nodes: t_path = [ os.path.join(role, mod) for mod in mods if os.path.exists( os.path.join( app_abs_path(self.main_conf.module), role, mod)) ] t_path.extend([ os.path.join("common", mod) for mod in mods if os.path.exists( os.path.join( app_abs_path(self.main_conf.module), "common", mod)) ]) modules[role] = t_path else: for role in match_nodes: role_mod_path = os.path.join( app_abs_path(self.main_conf.module), role) t_path = [ os.path.join(role, mod) for mod in os.listdir(role_mod_path) if mod.endswith(".py") ] common_mod_path = os.path.join( app_abs_path(self.main_conf.module), "common") t_path.extend([ os.path.join("common", mod) for mod in os.listdir(common_mod_path) if mod.endswith(".py") ]) modules[role] = t_path copy_pair = [] for role in modules: for mod in modules[role]: mod_path = os.path.join( app_abs_path(self.main_conf.module), mod) fid = fscli.upload(mod_path) copy_pair.append((role, fid, mod)) kwargs["copy_pair"] = copy_pair data = { "env": self.env, "payload": { "jid": self.jid, "cmd": cmd, "args": args, "kwargs": kwargs, "status": "READY", "timeout": self.timeout, "retry_times": self.retry_times } } if nthread != -1: data["payload"]["nthread"] = nthread send_ret = {} for role in match_nodes: for node_name in match_nodes[role]: job_data = deepcopy(data) send_ret.update({ "%s.%s" % (role, node_name): self._send_job(job_data, role, node_name) }) if wait_timeout: rets = {} @iTimeout(wait_timeout) def _return(nodes, job_rets): while 1: try: for r in nodes: for n in nodes[r]: job_ret = self.get_job(r, n, self.jid) i_ret = job_ret["payload"].get("return", "") if not i_ret: raise SwallAgentError("wait") if job_rets.get(r): job_rets[r].update({n: i_ret}) else: job_rets[r] = {n: i_ret} except SwallAgentError: time.sleep(0.1) else: break try: _return(match_nodes, rets) except Timeout, e: log.error(e) return { "retcode": 1, "extra_data": rets, "msg": "get result complete!" }