Ejemplo n.º 1
0
 def upload(self, upload_path):
     """
     上传文件
     @param upload_path string:本地文件路径
     @return string:remote file path if success else ""
     """
     if not upload_path:
         return ""
     fid = checksum(upload_path)
     max_retry = self.fs_conf.fs_failtry
     index = 1
     while index <= int(max_retry):
         ret = call(
             "RSYNC_PASSWORD=%s rsync -a --port=%s --partial %s %s@%s::swall_fs/%s" %
             (
                 self.fs_conf.fs_pass,
                 self.fs_conf.fs_port,
                 upload_path,
                 self.fs_conf.fs_user,
                 self.fs_conf.fs_host,
                 fid
             ),
             shell=True
         )
         if ret != 0:
             time.sleep(5)
         else:
             break
         index += 1
     if index <= int(max_retry):
         return fid
     else:
         return ""
Ejemplo n.º 2
0
 def upload(self, upload_path):
     """
     上传文件
     @param upload_path string:本地文件路径
     @return string:remote file path if success else ""
     """
     if not upload_path:
         return ""
     fid = checksum(upload_path)
     max_retry = self.fs_conf.fs_failtry
     index = 1
     while index <= int(max_retry):
         ret = call(
             "RSYNC_PASSWORD=%s rsync -a --port=%s --partial %s %s@%s::swall_fs/%s"
             % (self.fs_conf.fs_pass, self.fs_conf.fs_port, upload_path,
                self.fs_conf.fs_user, self.fs_conf.fs_host, fid),
             shell=True)
         if ret != 0:
             time.sleep(5)
         else:
             break
         index += 1
     if index <= int(max_retry):
         return fid
     else:
         return ""
Ejemplo n.º 3
0
    def get_job(self, role, node_name, jid):
        """
        获取任务
        @param role string:角色
        @param node_name string:节点名称
        @param jid string:任务id
        @return dict:a job info
        """
        ret = {}
        try:
            node_path = os.path.join(self.zookeeper_conf.nodes, role, node_name, "jobs", jid)
            data = self.zkconn.get(node_path)[0]
            data = msgpack.loads(data)
            env = data.get("env")
            if env == "aes":
                key_str = self.main_conf.token
                crypt = Crypt(key_str)
                data["payload"] = crypt.loads(data.get("payload"))
            payload = data["payload"]
            if payload["cmd"] == "sys.get" and payload["status"] == "FINISH" and payload["return"] != "":
                if payload["args"][0] != "help":
                    fid = payload["return"]
                    if "local_path" in payload["kwargs"] and "remote_path" in payload["kwargs"]:
                        local_path = payload["kwargs"]["local_path"]
                        remote_path = payload["kwargs"]["remote_path"]
                    else:
                        local_path = payload["args"][1]
                        remote_path = payload["args"][0]
                    stat = payload["kwargs"].get("stat")
                    if local_path.endswith('/') or os.path.isdir(local_path):
                        local_path = os.path.join(local_path, os.path.basename(remote_path))
                    if checksum(local_path) != fid:
                        if not check_cache(app_abs_path(self.main_conf.cache), fid):
                            FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
                            fscli = FsClient(self.fs_conf)
                            fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid))

                        if check_cache(app_abs_path(self.main_conf.cache), fid):
                            if not make_dirs(os.path.dirname(local_path)):
                                log.error("创建目标目录:%s失败" % local_path)
                            if cp(os.path.join(app_abs_path(self.main_conf.cache), fid), local_path, stat):
                                payload["return"] = local_path
                            else:
                                payload["return"] = ""
                    else:
                        payload["return"] = local_path
            ret = data

        except (ZKClientError, KeyboardInterrupt), e:
            log.error(e.message)
Ejemplo n.º 4
0
Archivo: job.py Proyecto: aihoney/swall
    def get_job(self, job_data):
        """
        获取任务
        @param node_name string:节点名称
        @param jid string:任务id
        @return dict:a job info
        """
        ret = {}
        key_str = self.main_conf.token
        crypt = Crypt(key_str)
        try:
            rets = self.mq.mget_job(job_data)
            for node, data in rets.items():
                if data:
                    env = data.get("env")
                    if env == "aes":
                        data["payload"] = crypt.loads(data.get("payload"))
                    payload = data["payload"]
                    if payload["cmd"] == "sys.get" and payload["status"] == "FINISH" and payload["return"] != "":
                        if payload["args"][0] != "help":
                            fid = payload["return"]
                            if "local_path" in payload["kwargs"] and "remote_path" in payload["kwargs"]:
                                local_path = payload["kwargs"]["local_path"]
                                remote_path = payload["kwargs"]["remote_path"]
                            else:
                                local_path = payload["args"][1]
                                remote_path = payload["args"][0]
                            stat = payload["kwargs"].get("stat")
                            if local_path.endswith('/') or os.path.isdir(local_path):
                                local_path = os.path.join(local_path, os.path.basename(remote_path))
                            if checksum(local_path) != fid:
                                if not check_cache(app_abs_path(self.main_conf.cache), fid):
                                    FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
                                                            ftype=self.fs_conf.fs_type)
                                    fscli = FsClient(self.fs_conf)
                                    fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid))

                                if check_cache(app_abs_path(self.main_conf.cache), fid):
                                    if not make_dirs(os.path.dirname(local_path)):
                                        log.error("创建目标目录:%s失败" % local_path)
                                    if cp(os.path.join(app_abs_path(self.main_conf.cache), fid), local_path, stat):
                                        payload["return"] = local_path
                                    else:
                                        payload["return"] = ""
                            else:
                                payload["return"] = local_path
                ret[node] = data

        except Exception, e:
            log.error(traceback.format_exc())
Ejemplo n.º 5
0
 def upload(self, upload_path):
     """
     上传文件
     @param upload_path string:本地文件路径
     @return string:remote file path if success else ""
     """
     fid = ""
     if os.path.exists(upload_path):
         fid = checksum(upload_path)
         dist = os.path.join(self.fs_conf.fs_tmp_dir, fid)
         if not self.exists(dist):
             try:
                 self.sftp.put(upload_path, dist)
             except IOError, er:
                 log.error(er)
                 return ""
Ejemplo n.º 6
0
 def upload(self, upload_path):
     """
     上传文件
     @param upload_path string:本地文件路径
     @return string:remote file path if success else ""
     """
     fid = ""
     if os.path.exists(upload_path):
         fid = checksum(upload_path)
         dist = os.path.join(self.fs_conf.fs_tmp_dir, fid)
         if not self.exists(dist):
             try:
                 self.sftp.put(upload_path, dist)
             except IOError, er:
                 log.error(er)
                 return ""
Ejemplo n.º 7
0
 def upload(self, upload_path):
     """
     上传文件
     @param upload_path string:本地文件路径
     @return string:remote file path if success else ""
     """
     if not upload_path:
         return ""
     fid = checksum(upload_path)
     if self.exists(fid):
         return 1
     ftp_path = os.path.join(self.fs_conf.fs_tmp_dir, fid)
     try:
         f = open(upload_path, 'rb')
         self.ftp.storbinary('STOR %s' % ftp_path, f, BLOCK_SIZE)
     except :
         log.error(traceback.format_exc())
         return 0
     return 1
Ejemplo n.º 8
0
 def upload(self, upload_path):
     """
     上传文件
     @param upload_path string:本地文件路径
     @return string:remote file path if success else ""
     """
     if not upload_path:
         return ""
     fid = checksum(upload_path)
     if self.exists(fid):
         return 1
     ftp_path = os.path.join(self.fs_conf.fs_tmp_dir, fid)
     try:
         f = open(upload_path, 'rb')
         self.ftp.storbinary('STOR %s' % ftp_path, f, BLOCK_SIZE)
     except :
         log.error(traceback.format_exc())
         return 0
     return 1
Ejemplo n.º 9
0
    def get_job(self, job_data):
        """
        获取任务
        @param node_name string:节点名称
        @param jid string:任务id
        @return dict:a job info
        """
        ret = {}
        key_str = self.main_conf.token
        crypt = Crypt(key_str)
        try:
            rets = self.mq.mget_job(job_data)
            for node, data in rets.items():
                if data:
                    env = data.get("env")
                    if env == "aes":
                        data["payload"] = crypt.loads(data.get("payload"))
                    payload = data["payload"]
                    if payload["cmd"] == "sys.get" and payload[
                            "status"] == "FINISH" and payload["return"] != "":
                        if payload["args"][0] != "help":
                            fid = payload["return"]
                            if "local_path" in payload[
                                    "kwargs"] and "remote_path" in payload[
                                        "kwargs"]:
                                local_path = payload["kwargs"]["local_path"]
                                remote_path = payload["kwargs"]["remote_path"]
                            else:
                                local_path = payload["args"][1]
                                remote_path = payload["args"][0]
                            stat = payload["kwargs"].get("stat")
                            if local_path.endswith('/') or os.path.isdir(
                                    local_path):
                                local_path = os.path.join(
                                    local_path, os.path.basename(remote_path))
                            if checksum(local_path) != fid:
                                if not check_cache(
                                        app_abs_path(self.main_conf.cache),
                                        fid):
                                    FsClient = load_fclient(
                                        app_abs_path(self.main_conf.fs_plugin),
                                        ftype=self.fs_conf.fs_type)
                                    fscli = FsClient(self.fs_conf)
                                    fscli.download(
                                        fid,
                                        os.path.join(
                                            app_abs_path(self.main_conf.cache),
                                            fid))

                                if check_cache(
                                        app_abs_path(self.main_conf.cache),
                                        fid):
                                    if not make_dirs(
                                            os.path.dirname(local_path)):
                                        log.error("创建目标目录:%s失败" % local_path)
                                    if cp(
                                            os.path.join(
                                                app_abs_path(
                                                    self.main_conf.cache),
                                                fid), local_path, stat):
                                        payload["return"] = local_path
                                    else:
                                        payload["return"] = ""
                            else:
                                payload["return"] = local_path
                ret[node] = data

        except Exception, e:
            log.error(traceback.format_exc())
Ejemplo n.º 10
0
    def get_job(self, role, node_name, jid):
        """
        获取任务
        @param role string:角色
        @param node_name string:节点名称
        @param jid string:任务id
        @return dict:a job info
        """
        ret = {}
        try:
            node_path = os.path.join(self.zookeeper_conf.nodes, role,
                                     node_name, "jobs", jid)
            data = self.zkconn.get(node_path)[0]
            data = msgpack.loads(data)
            env = data.get("env")
            if env == "aes":
                key_str = self.main_conf.token
                crypt = Crypt(key_str)
                data["payload"] = crypt.loads(data.get("payload"))
            payload = data["payload"]
            if payload["cmd"] == "sys.get" and payload[
                    "status"] == "FINISH" and payload["return"] != "":
                if payload["args"][0] != "help":
                    fid = payload["return"]
                    if "local_path" in payload[
                            "kwargs"] and "remote_path" in payload["kwargs"]:
                        local_path = payload["kwargs"]["local_path"]
                        remote_path = payload["kwargs"]["remote_path"]
                    else:
                        local_path = payload["args"][1]
                        remote_path = payload["args"][0]
                    stat = payload["kwargs"].get("stat")
                    if local_path.endswith('/') or os.path.isdir(local_path):
                        local_path = os.path.join(
                            local_path, os.path.basename(remote_path))
                    if checksum(local_path) != fid:
                        if not check_cache(app_abs_path(self.main_conf.cache),
                                           fid):
                            FsClient = load_fclient(app_abs_path(
                                self.main_conf.fs_plugin),
                                                    ftype=self.fs_conf.fs_type)
                            fscli = FsClient(self.fs_conf)
                            fscli.download(
                                fid,
                                os.path.join(
                                    app_abs_path(self.main_conf.cache), fid))

                        if check_cache(app_abs_path(self.main_conf.cache),
                                       fid):
                            if not make_dirs(os.path.dirname(local_path)):
                                log.error("创建目标目录:%s失败" % local_path)
                            if cp(
                                    os.path.join(
                                        app_abs_path(self.main_conf.cache),
                                        fid), local_path, stat):
                                payload["return"] = local_path
                            else:
                                payload["return"] = ""
                    else:
                        payload["return"] = local_path
            ret = data

        except (ZKClientError, KeyboardInterrupt), e:
            log.error(e.message)