def get_job(self, role, node_name, jid): """ 获取任务 @param role string:角色 @param node_name string:节点名称 @param jid string:任务id @return dict:a job info """ ret = {} try: node_path = os.path.join(self.zookeeper_conf.nodes, role, node_name, "jobs", jid) data = self.zkconn.get(node_path)[0] data = msgpack.loads(data) env = data.get("env") if env == "aes": key_str = self.main_conf.token crypt = Crypt(key_str) data["payload"] = crypt.loads(data.get("payload")) payload = data["payload"] if payload["cmd"] == "sys.get" and payload["status"] == "FINISH" and payload["return"] != "": if payload["args"][0] != "help": fid = payload["return"] if "local_path" in payload["kwargs"] and "remote_path" in payload["kwargs"]: local_path = payload["kwargs"]["local_path"] remote_path = payload["kwargs"]["remote_path"] else: local_path = payload["args"][1] remote_path = payload["args"][0] stat = payload["kwargs"].get("stat") if local_path.endswith('/') or os.path.isdir(local_path): local_path = os.path.join(local_path, os.path.basename(remote_path)) if checksum(local_path) != fid: if not check_cache(app_abs_path(self.main_conf.cache), fid): FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid)) if check_cache(app_abs_path(self.main_conf.cache), fid): if not make_dirs(os.path.dirname(local_path)): log.error("创建目标目录:%s失败" % local_path) if cp(os.path.join(app_abs_path(self.main_conf.cache), fid), local_path, stat): payload["return"] = local_path else: payload["return"] = "" else: payload["return"] = local_path ret = data except (ZKClientError, KeyboardInterrupt), e: log.error(e.message)
def _get(self, *args, **kwargs): """ def get(*args, **kwargs) -> 从远程获取文件 @param args list:支持位置参数,例如 sys.get /tmp/src.tar.gz /etc/src.tar.gz @param kwargs dict:支持关键字参数,例如sys.get remote_path=/tmp/src.tar.gz local_path=/etc/src.tar.gz @return string:local_path """ if "local_path" in kwargs and "remote_path" in kwargs: remote_path = kwargs["remote_path"] else: remote_path = args[0] FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) return fscli.upload(remote_path)
def get_job(self, job_data): """ 获取任务 @param node_name string:节点名称 @param jid string:任务id @return dict:a job info """ ret = {} key_str = self.main_conf.token crypt = Crypt(key_str) try: rets = self.mq.mget_job(job_data) for node, data in rets.items(): if data: env = data.get("env") if env == "aes": data["payload"] = crypt.loads(data.get("payload")) payload = data["payload"] if payload["cmd"] == "sys.get" and payload["status"] == "FINISH" and payload["return"] != "": if payload["args"][0] != "help": fid = payload["return"] if "local_path" in payload["kwargs"] and "remote_path" in payload["kwargs"]: local_path = payload["kwargs"]["local_path"] remote_path = payload["kwargs"]["remote_path"] else: local_path = payload["args"][1] remote_path = payload["args"][0] stat = payload["kwargs"].get("stat") if local_path.endswith('/') or os.path.isdir(local_path): local_path = os.path.join(local_path, os.path.basename(remote_path)) if checksum(local_path) != fid: if not check_cache(app_abs_path(self.main_conf.cache), fid): FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid)) if check_cache(app_abs_path(self.main_conf.cache), fid): if not make_dirs(os.path.dirname(local_path)): log.error("创建目标目录:%s失败" % local_path) if cp(os.path.join(app_abs_path(self.main_conf.cache), fid), local_path, stat): payload["return"] = local_path else: payload["return"] = "" else: payload["return"] = local_path ret[node] = data except Exception, e: log.error(traceback.format_exc())
def _copy(self, *args, **kwargs): """ def copy(*args, **kwargs) -> 拷贝文件到远程 可以增加一个ret_type=full,支持返回文件名 @param args list:支持位置参数,例如 sys.copy /etc/src.tar.gz /tmp/src.tar.gz ret_type=full @param kwargs dict:支持关键字参数,例如sys.copy local_path=/etc/src.tar.gz remote_path=/tmp/src.tar.gz ret_type=full @return int:1 if success else 0 """ if "path_pair" in kwargs and "remote_path" in kwargs: fid, file_name = kwargs["path_pair"].split(',') remote_path = kwargs["remote_path"] make_path = kwargs.get("make_path", 1) else: fid, file_name = args[0].split(',') remote_path = args[1] make_path = args[2] if len(args) >= 3 else 1 stat = kwargs.get("stat") ret_type = kwargs.get("ret_type") if os.path.isdir(remote_path) or remote_path.endswith('/'): remote_path = os.path.join(remote_path, file_name) try: if int(make_path): make_dirs(os.path.dirname(remote_path)) else: if not os.path.exists(os.path.dirname(remote_path)): return "" except: log.info(traceback.format_exc()) #如果cache中没有对应的文件,则先从fs中拷贝过来 if not check_cache(app_abs_path(self.main_conf.cache), fid): FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) fscli.download( fid, os.path.join(app_abs_path(self.main_conf.cache), fid)) #从cache目录中拷贝文件到目标 ret = cp(os.path.join(app_abs_path(self.main_conf.cache), fid), remote_path, stat) if ret_type == "full": return remote_path if ret else "" else: return ret
def _copy(self, *args, **kwargs): """ def copy(*args, **kwargs) -> 拷贝文件到远程 可以增加一个ret_type=full,支持返回文件名 @param args list:支持位置参数,例如 sys.copy /etc/src.tar.gz /tmp/src.tar.gz ret_type=full @param kwargs dict:支持关键字参数,例如sys.copy local_path=/etc/src.tar.gz remote_path=/tmp/src.tar.gz ret_type=full @return int:1 if success else 0 """ if "path_pair" in kwargs and "remote_path" in kwargs: fid, file_name = kwargs["path_pair"].split(',') remote_path = kwargs["remote_path"] make_path = kwargs.get("make_path", 1) else: fid, file_name = args[0].split(',') remote_path = args[1] make_path = args[2] if len(args) >= 3 else 1 stat = kwargs.get("stat") ret_type = kwargs.get("ret_type") if os.path.isdir(remote_path) or remote_path.endswith('/'): remote_path = os.path.join(remote_path, file_name) try: if int(make_path): make_dirs(os.path.dirname(remote_path)) else: if not os.path.exists(os.path.dirname(remote_path)): return "" except: log.info(traceback.format_exc()) #如果cache中没有对应的文件,则先从fs中拷贝过来 if not check_cache(app_abs_path(self.main_conf.cache), fid): FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid)) #从cache目录中拷贝文件到目标 ret = cp(os.path.join(app_abs_path(self.main_conf.cache), fid), remote_path, stat) if ret_type == "full": return remote_path if ret else "" else: return ret
dist = os.path.join(self.fs_conf.fs_tmp_dir, fid) self.sftp.stat(dist) return 1 except IOError: return 0 def download(self, fid, to_path): """ 下载文件 @param filename string:需要下载的文件路径,远程文件 @param to_path string:存放本地的文件路径 @return int:1 if success else 0 """ try: dist = os.path.join(self.fs_conf.fs_tmp_dir, fid) self.sftp.get(dist, to_path) return 1 except IOError, er: log.error(er) return 0 if __name__ == "__main__": client = load_fclient("~/Documents/works/git/swall/plugins/fservice", ftype="rsync") print client scli = client("~/Documents/works/git/swall/conf/swall.conf") print scli print scli.upload("/etc/services") print scli.download("f9f1d3bc559b817e74c13efc3fd1172fbe170d37", "/tmp/a.txt")
def submit_job(self, cmd, roles, nregex, nexclude=None, args=[], kwargs={}, wait_timeout=0, nthread=-1): """ 提交任务 @param cmd string:需要执行的命令 @param roles string:需要执行的节点类型,如game、admin、server等,多个用|分隔 @param nregex string:节点匹配正则表达式 @param nexclude string:排除节点正则,会从nregex结果排除掉 @param args list:传给cmd命令的位置参数 @param kwargs dict:传给cmd的位置参数 @param wait_timeout int:等待结果的时间 @param nthread int:单个机器上面执行任务的并发数量 @return dict:{ "retcode": 返回值 "extra_data": 其他信息, "msg": 提示信息, } """ self._gen_jid() match_nodes = {} for role in re.split(r"[|,]", roles): match = self.keeper.get_nodes_by_regex(role, nregex, nexclude) if match: match_nodes[role] = match if not match_nodes: log.warn("0 node match for %s [%s]" % (self.jid, cmd)) return { "retcode": 1, "extra_data": {}, "msg": "send_job complete,0 node match" } if cmd == "sys.copy": if "help" not in args: FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) if "local_path" in kwargs and "remote_path" in kwargs: local_path = kwargs["local_path"] else: local_path = args[0] fid = fscli.upload(local_path) if "local_path" in kwargs and "remote_path" in kwargs: kwargs["path_pair"] = "%s,%s" % (fid, os.path.basename(local_path)) else: args[0] = "%s,%s" % (fid, os.path.basename(local_path)) if cmd == "sys.rsync_module": if not args or args[0] != "help": FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) if "mods" in kwargs: mods = kwargs["mods"].split(',') elif args: mods = args else: pass modules = {} if "mods" in kwargs or args: for role in match_nodes: t_path = [ os.path.join(role, mod) for mod in mods if os.path.exists(os.path.join(app_abs_path(self.main_conf.module), role, mod)) ] t_path.extend([ os.path.join("common", mod) for mod in mods if os.path.exists(os.path.join(app_abs_path(self.main_conf.module), "common", mod)) ]) modules[role] = t_path else: for role in match_nodes: role_mod_path = os.path.join(app_abs_path(self.main_conf.module), role) t_path = [ os.path.join(role, mod) for mod in os.listdir(role_mod_path) if mod.endswith(".py") ] common_mod_path = os.path.join(app_abs_path(self.main_conf.module), "common") t_path.extend( [ os.path.join("common", mod) for mod in os.listdir(common_mod_path) if mod.endswith(".py") ] ) modules[role] = t_path copy_pair = [] for role in modules: for mod in modules[role]: mod_path = os.path.join(app_abs_path(self.main_conf.module), mod) fid = fscli.upload(mod_path) copy_pair.append((role, fid, mod)) kwargs["copy_pair"] = copy_pair data = { "env": self.env, "payload": { "jid": self.jid, "cmd": cmd, "args": args, "kwargs": kwargs, "status": "READY", "timeout": self.timeout, "retry_times": self.retry_times } } if nthread != -1: data["payload"]["nthread"] = nthread send_ret = {} for role in match_nodes: for node_name in match_nodes[role]: job_data = deepcopy(data) send_ret.update({"%s.%s" % (role, node_name): self._send_job(job_data, role, node_name)}) if wait_timeout: rets = {} @iTimeout(wait_timeout) def _return(nodes, job_rets): while 1: try: for r in nodes: for n in nodes[r]: job_ret = self.get_job(r, n, self.jid) i_ret = job_ret["payload"].get("return", "") if not i_ret: raise SwallAgentError("wait") if job_rets.get(r): job_rets[r].update({n: i_ret}) else: job_rets[r] = {n: i_ret} except SwallAgentError: time.sleep(0.1) else: break try: _return(match_nodes, rets) except Timeout, e: log.error(e) return { "retcode": 1, "extra_data": rets, "msg": "get result complete!" }
index = 1 while index <= int(max_retry): ret = call( "RSYNC_PASSWORD=%s rsync -a --port=%s --partial %s@%s::swall_fs/%s %s" % ( self.fs_conf.fs_pass, self.fs_conf.fs_port, self.fs_conf.fs_user, self.fs_conf.fs_host, fid, to_path ), shell=True ) if ret != 0: time.sleep(5) else: break index += 1 if index <= int(max_retry): return 1 else: return 0 if __name__ == "__main__": client = load_fclient("~/Documents/works/git/swall/plugins/fservice", ftype="rsync") print client scli = client("~/Documents/works/git/swall/conf/swall.conf") print scli print scli.upload("/etc/services") print scli.download("f9f1d3bc559b817e74c13efc3fd1172fbe170d37","/tmp/a.txt")
def submit_job(self, cmd, nregex, nexclude=None, args=[], kwargs={}, wait_timeout=0, nthread=-1): """ 提交任务 @param cmd string:需要执行的命令 @param nregex string:节点匹配正则表达式 @param nexclude string:排除节点正则,会从nregex结果排除掉 @param args list:传给cmd命令的位置参数 @param kwargs dict:传给cmd的位置参数 @param wait_timeout int:等待结果的时间 @param nthread int:单个机器上面执行任务的并发数量 @return dict:{ "retcode": 返回值 "extra_data": 其他信息, "msg": 提示信息, } """ self._gen_jid() match_nodes = [] match = self.keeper.get_nodes_by_regex(nregex, nexclude) if match: match_nodes = match if not match_nodes: log.warn("0 node match for %s [%s]" % (self.jid, cmd)) return { "retcode": 1, "extra_data": {}, "msg": "send_job complete,0 node match" } if cmd == "sys.copy": if "help" not in args: FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) if "local_path" in kwargs and "remote_path" in kwargs: local_path = kwargs["local_path"] else: local_path = args[0] fid = fscli.upload(local_path) if "local_path" in kwargs and "remote_path" in kwargs: kwargs["path_pair"] = "%s,%s" % ( fid, os.path.basename(local_path)) else: args[0] = "%s,%s" % (fid, os.path.basename(local_path)) if cmd == "sys.rsync_module": if not args or args[0] != "help": FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) modules = [ mod for mod in os.listdir(app_abs_path(self.main_conf.module)) if mod.endswith(".py") ] copy_pair = [] for mod in modules: mod_path = os.path.join( app_abs_path(self.main_conf.module), mod) fid = fscli.upload(mod_path) copy_pair.append((fid, mod)) kwargs["copy_pair"] = copy_pair data = { "env": self.env, "payload": { "jid": self.jid, "cmd": cmd, "args": args, "kwargs": kwargs, "status": "READY", "timeout": self.timeout, "retry_times": self.retry_times } } if nthread != -1: data["payload"]["nthread"] = nthread node_data = [] for node_name in match_nodes: job_data = deepcopy(data) node_data.append((job_data, node_name)) ret = self._send_job(node_data) send_ret = {n: ret for n in match_nodes} if wait_timeout: rets = {} @iTimeout(wait_timeout) def _return(nodes, job_rets): while 1: job_ret = self.get_job([(n, self.jid) for n in nodes]) for node, ret_ in job_ret.iteritems(): if ret_: i_ret = ret_["payload"].get("return") if i_ret is not None: if job_rets: job_rets.update({node: i_ret}) else: job_rets = {node: i_ret} is_wait = False for ret_ in job_ret.itervalues(): if not ret_: is_wait = True else: i_ret = ret_["payload"].get("return") if i_ret is None: is_wait = True if is_wait: continue else: break try: _return(match_nodes, rets) except Timeout, e: log.error(e) return { "retcode": 1, "extra_data": rets, "msg": "get result complete!" }
def get_job(self, job_data): """ 获取任务 @param node_name string:节点名称 @param jid string:任务id @return dict:a job info """ ret = {} key_str = self.main_conf.token crypt = Crypt(key_str) try: rets = self.mq.mget_job(job_data) for node, data in rets.items(): if data: env = data.get("env") if env == "aes": data["payload"] = crypt.loads(data.get("payload")) payload = data["payload"] if payload["cmd"] == "sys.get" and payload[ "status"] == "FINISH" and payload["return"] != "": if payload["args"][0] != "help": fid = payload["return"] if "local_path" in payload[ "kwargs"] and "remote_path" in payload[ "kwargs"]: local_path = payload["kwargs"]["local_path"] remote_path = payload["kwargs"]["remote_path"] else: local_path = payload["args"][1] remote_path = payload["args"][0] stat = payload["kwargs"].get("stat") if local_path.endswith('/') or os.path.isdir( local_path): local_path = os.path.join( local_path, os.path.basename(remote_path)) if checksum(local_path) != fid: if not check_cache( app_abs_path(self.main_conf.cache), fid): FsClient = load_fclient( app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) fscli.download( fid, os.path.join( app_abs_path(self.main_conf.cache), fid)) if check_cache( app_abs_path(self.main_conf.cache), fid): if not make_dirs( os.path.dirname(local_path)): log.error("创建目标目录:%s失败" % local_path) if cp( os.path.join( app_abs_path( self.main_conf.cache), fid), local_path, stat): payload["return"] = local_path else: payload["return"] = "" else: payload["return"] = local_path ret[node] = data except Exception, e: log.error(traceback.format_exc())
def submit_job(self, cmd, nregex, nexclude=None, args=[], kwargs={}, wait_timeout=0, nthread=-1): """ 提交任务 @param cmd string:需要执行的命令 @param nregex string:节点匹配正则表达式 @param nexclude string:排除节点正则,会从nregex结果排除掉 @param args list:传给cmd命令的位置参数 @param kwargs dict:传给cmd的位置参数 @param wait_timeout int:等待结果的时间 @param nthread int:单个机器上面执行任务的并发数量 @return dict:{ "retcode": 返回值 "extra_data": 其他信息, "msg": 提示信息, } """ self._gen_jid() match_nodes = [] match = self.keeper.get_nodes_by_regex(nregex, nexclude) if match: match_nodes = match if not match_nodes: log.warn("0 node match for %s [%s]" % (self.jid, cmd)) return { "retcode": 1, "extra_data": {}, "msg": "send_job complete,0 node match" } if cmd == "sys.copy": if "help" not in args: FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) if "local_path" in kwargs and "remote_path" in kwargs: local_path = kwargs["local_path"] else: local_path = args[0] fid = fscli.upload(local_path) if "local_path" in kwargs and "remote_path" in kwargs: kwargs["path_pair"] = "%s,%s" % (fid, os.path.basename(local_path)) else: args[0] = "%s,%s" % (fid, os.path.basename(local_path)) if cmd == "sys.rsync_module": if not args or args[0] != "help": FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) modules = [mod for mod in os.listdir(app_abs_path(self.main_conf.module)) if mod.endswith(".py")] copy_pair = [] for mod in modules: mod_path = os.path.join(app_abs_path(self.main_conf.module), mod) fid = fscli.upload(mod_path) copy_pair.append((fid, mod)) kwargs["copy_pair"] = copy_pair data = { "env": self.env, "payload": { "jid": self.jid, "cmd": cmd, "args": args, "kwargs": kwargs, "status": "READY", "timeout": self.timeout, "retry_times": self.retry_times } } if nthread != -1: data["payload"]["nthread"] = nthread node_data = [] for node_name in match_nodes: job_data = deepcopy(data) node_data.append((job_data, node_name)) ret = self._send_job(node_data) send_ret = {n: ret for n in match_nodes} if wait_timeout: rets = {} @iTimeout(wait_timeout) def _return(nodes, job_rets): while 1: job_ret = self.get_job([(n, self.jid) for n in nodes]) for node, ret_ in job_ret.iteritems(): if ret_: i_ret = ret_["payload"].get("return") if i_ret is not None: if job_rets: job_rets.update({node: i_ret}) else: job_rets = {node: i_ret} is_wait = False for ret_ in job_ret.itervalues(): if not ret_: is_wait = True else: i_ret = ret_["payload"].get("return") if i_ret is None: is_wait = True if is_wait: continue else: break try: _return(match_nodes, rets) except Timeout, e: log.error(e) return { "retcode": 1, "extra_data": rets, "msg": "get result complete!" }
def submit_job(self, cmd, roles, nregex, nexclude=None, args=[], kwargs={}, wait_timeout=0, nthread=-1): """ 提交任务 @param cmd string:需要执行的命令 @param roles string:需要执行的节点类型,如game、admin、server等,多个用|分隔 @param nregex string:节点匹配正则表达式 @param nexclude string:排除节点正则,会从nregex结果排除掉 @param args list:传给cmd命令的位置参数 @param kwargs dict:传给cmd的位置参数 @param wait_timeout int:等待结果的时间 @param nthread int:单个机器上面执行任务的并发数量 @return dict:{ "retcode": 返回值 "extra_data": 其他信息, "msg": 提示信息, } """ self._gen_jid() match_nodes = {} for role in re.split(r"[|,]", roles): match = self.keeper.get_nodes_by_regex(role, nregex, nexclude) if match: match_nodes[role] = match if not match_nodes: log.warn("0 node match for %s [%s]" % (self.jid, cmd)) return { "retcode": 1, "extra_data": {}, "msg": "send_job complete,0 node match" } if cmd == "sys.copy": if "help" not in args: FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) if "local_path" in kwargs and "remote_path" in kwargs: local_path = kwargs["local_path"] else: local_path = args[0] fid = fscli.upload(local_path) if "local_path" in kwargs and "remote_path" in kwargs: kwargs["path_pair"] = "%s,%s" % ( fid, os.path.basename(local_path)) else: args[0] = "%s,%s" % (fid, os.path.basename(local_path)) if cmd == "sys.rsync_module": if not args or args[0] != "help": FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) if "mods" in kwargs: mods = kwargs["mods"].split(',') elif args: mods = args else: pass modules = {} if "mods" in kwargs or args: for role in match_nodes: t_path = [ os.path.join(role, mod) for mod in mods if os.path.exists( os.path.join( app_abs_path(self.main_conf.module), role, mod)) ] t_path.extend([ os.path.join("common", mod) for mod in mods if os.path.exists( os.path.join( app_abs_path(self.main_conf.module), "common", mod)) ]) modules[role] = t_path else: for role in match_nodes: role_mod_path = os.path.join( app_abs_path(self.main_conf.module), role) t_path = [ os.path.join(role, mod) for mod in os.listdir(role_mod_path) if mod.endswith(".py") ] common_mod_path = os.path.join( app_abs_path(self.main_conf.module), "common") t_path.extend([ os.path.join("common", mod) for mod in os.listdir(common_mod_path) if mod.endswith(".py") ]) modules[role] = t_path copy_pair = [] for role in modules: for mod in modules[role]: mod_path = os.path.join( app_abs_path(self.main_conf.module), mod) fid = fscli.upload(mod_path) copy_pair.append((role, fid, mod)) kwargs["copy_pair"] = copy_pair data = { "env": self.env, "payload": { "jid": self.jid, "cmd": cmd, "args": args, "kwargs": kwargs, "status": "READY", "timeout": self.timeout, "retry_times": self.retry_times } } if nthread != -1: data["payload"]["nthread"] = nthread send_ret = {} for role in match_nodes: for node_name in match_nodes[role]: job_data = deepcopy(data) send_ret.update({ "%s.%s" % (role, node_name): self._send_job(job_data, role, node_name) }) if wait_timeout: rets = {} @iTimeout(wait_timeout) def _return(nodes, job_rets): while 1: try: for r in nodes: for n in nodes[r]: job_ret = self.get_job(r, n, self.jid) i_ret = job_ret["payload"].get("return", "") if not i_ret: raise SwallAgentError("wait") if job_rets.get(r): job_rets[r].update({n: i_ret}) else: job_rets[r] = {n: i_ret} except SwallAgentError: time.sleep(0.1) else: break try: _return(match_nodes, rets) except Timeout, e: log.error(e) return { "retcode": 1, "extra_data": rets, "msg": "get result complete!" }
def get_job(self, role, node_name, jid): """ 获取任务 @param role string:角色 @param node_name string:节点名称 @param jid string:任务id @return dict:a job info """ ret = {} try: node_path = os.path.join(self.zookeeper_conf.nodes, role, node_name, "jobs", jid) data = self.zkconn.get(node_path)[0] data = msgpack.loads(data) env = data.get("env") if env == "aes": key_str = self.main_conf.token crypt = Crypt(key_str) data["payload"] = crypt.loads(data.get("payload")) payload = data["payload"] if payload["cmd"] == "sys.get" and payload[ "status"] == "FINISH" and payload["return"] != "": if payload["args"][0] != "help": fid = payload["return"] if "local_path" in payload[ "kwargs"] and "remote_path" in payload["kwargs"]: local_path = payload["kwargs"]["local_path"] remote_path = payload["kwargs"]["remote_path"] else: local_path = payload["args"][1] remote_path = payload["args"][0] stat = payload["kwargs"].get("stat") if local_path.endswith('/') or os.path.isdir(local_path): local_path = os.path.join( local_path, os.path.basename(remote_path)) if checksum(local_path) != fid: if not check_cache(app_abs_path(self.main_conf.cache), fid): FsClient = load_fclient(app_abs_path( self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type) fscli = FsClient(self.fs_conf) fscli.download( fid, os.path.join( app_abs_path(self.main_conf.cache), fid)) if check_cache(app_abs_path(self.main_conf.cache), fid): if not make_dirs(os.path.dirname(local_path)): log.error("创建目标目录:%s失败" % local_path) if cp( os.path.join( app_abs_path(self.main_conf.cache), fid), local_path, stat): payload["return"] = local_path else: payload["return"] = "" else: payload["return"] = local_path ret = data except (ZKClientError, KeyboardInterrupt), e: log.error(e.message)