def upload(self, upload_path):
"""
上传文件
@param upload_path string:本地文件路径
@return string:remote file path if success else ""
"""
if not upload_path:
return ""
fid = checksum(upload_path)
max_retry = self.fs_conf.fs_failtry
index = 1
while index <= int(max_retry):
ret = call(
"RSYNC_PASSWORD=%s rsync -a --port=%s --partial %s %s@%s::swall_fs/%s" %
(
self.fs_conf.fs_pass,
self.fs_conf.fs_port,
upload_path,
self.fs_conf.fs_user,
self.fs_conf.fs_host,
fid
),
shell=True
)
if ret != 0:
time.sleep(5)
else:
break
index += 1
if index <= int(max_retry):
return fid
else:
return ""
def upload(self, upload_path):
"""
上传文件
@param upload_path string:本地文件路径
@return string:remote file path if success else ""
"""
if not upload_path:
return ""
fid = checksum(upload_path)
max_retry = self.fs_conf.fs_failtry
index = 1
while index <= int(max_retry):
ret = call(
"RSYNC_PASSWORD=%s rsync -a --port=%s --partial %s %s@%s::swall_fs/%s"
% (self.fs_conf.fs_pass, self.fs_conf.fs_port, upload_path,
self.fs_conf.fs_user, self.fs_conf.fs_host, fid),
shell=True)
if ret != 0:
time.sleep(5)
else:
break
index += 1
if index <= int(max_retry):
return fid
else:
return ""
def get_job(self, role, node_name, jid):
"""
获取任务
@param role string:角色
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
try:
node_path = os.path.join(self.zookeeper_conf.nodes, role, node_name, "jobs", jid)
data = self.zkconn.get(node_path)[0]
data = msgpack.loads(data)
env = data.get("env")
if env == "aes":
key_str = self.main_conf.token
crypt = Crypt(key_str)
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload["status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload["kwargs"] and "remote_path" in payload["kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(local_path):
local_path = os.path.join(local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(app_abs_path(self.main_conf.cache), fid):
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin), ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid))
if check_cache(app_abs_path(self.main_conf.cache), fid):
if not make_dirs(os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(os.path.join(app_abs_path(self.main_conf.cache), fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret = data
except (ZKClientError, KeyboardInterrupt), e:
log.error(e.message)
def get_job(self, job_data):
"""
获取任务
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
key_str = self.main_conf.token
crypt = Crypt(key_str)
try:
rets = self.mq.mget_job(job_data)
for node, data in rets.items():
if data:
env = data.get("env")
if env == "aes":
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload["status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload["kwargs"] and "remote_path" in payload["kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(local_path):
local_path = os.path.join(local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(app_abs_path(self.main_conf.cache), fid):
FsClient = load_fclient(app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(fid, os.path.join(app_abs_path(self.main_conf.cache), fid))
if check_cache(app_abs_path(self.main_conf.cache), fid):
if not make_dirs(os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(os.path.join(app_abs_path(self.main_conf.cache), fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret[node] = data
except Exception, e:
log.error(traceback.format_exc())
def upload(self, upload_path):
"""
上传文件
@param upload_path string:本地文件路径
@return string:remote file path if success else ""
"""
fid = ""
if os.path.exists(upload_path):
fid = checksum(upload_path)
dist = os.path.join(self.fs_conf.fs_tmp_dir, fid)
if not self.exists(dist):
try:
self.sftp.put(upload_path, dist)
except IOError, er:
log.error(er)
return ""
def upload(self, upload_path):
"""
上传文件
@param upload_path string:本地文件路径
@return string:remote file path if success else ""
"""
fid = ""
if os.path.exists(upload_path):
fid = checksum(upload_path)
dist = os.path.join(self.fs_conf.fs_tmp_dir, fid)
if not self.exists(dist):
try:
self.sftp.put(upload_path, dist)
except IOError, er:
log.error(er)
return ""
def upload(self, upload_path):
"""
上传文件
@param upload_path string:本地文件路径
@return string:remote file path if success else ""
"""
if not upload_path:
return ""
fid = checksum(upload_path)
if self.exists(fid):
return 1
ftp_path = os.path.join(self.fs_conf.fs_tmp_dir, fid)
try:
f = open(upload_path, 'rb')
self.ftp.storbinary('STOR %s' % ftp_path, f, BLOCK_SIZE)
except :
log.error(traceback.format_exc())
return 0
return 1
def upload(self, upload_path):
"""
上传文件
@param upload_path string:本地文件路径
@return string:remote file path if success else ""
"""
if not upload_path:
return ""
fid = checksum(upload_path)
if self.exists(fid):
return 1
ftp_path = os.path.join(self.fs_conf.fs_tmp_dir, fid)
try:
f = open(upload_path, 'rb')
self.ftp.storbinary('STOR %s' % ftp_path, f, BLOCK_SIZE)
except :
log.error(traceback.format_exc())
return 0
return 1
def get_job(self, job_data):
"""
获取任务
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
key_str = self.main_conf.token
crypt = Crypt(key_str)
try:
rets = self.mq.mget_job(job_data)
for node, data in rets.items():
if data:
env = data.get("env")
if env == "aes":
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload[
"status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload[
"kwargs"] and "remote_path" in payload[
"kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(
local_path):
local_path = os.path.join(
local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(
app_abs_path(self.main_conf.cache),
fid):
FsClient = load_fclient(
app_abs_path(self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(
fid,
os.path.join(
app_abs_path(self.main_conf.cache),
fid))
if check_cache(
app_abs_path(self.main_conf.cache),
fid):
if not make_dirs(
os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(
os.path.join(
app_abs_path(
self.main_conf.cache),
fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret[node] = data
except Exception, e:
log.error(traceback.format_exc())
def get_job(self, role, node_name, jid):
"""
获取任务
@param role string:角色
@param node_name string:节点名称
@param jid string:任务id
@return dict:a job info
"""
ret = {}
try:
node_path = os.path.join(self.zookeeper_conf.nodes, role,
node_name, "jobs", jid)
data = self.zkconn.get(node_path)[0]
data = msgpack.loads(data)
env = data.get("env")
if env == "aes":
key_str = self.main_conf.token
crypt = Crypt(key_str)
data["payload"] = crypt.loads(data.get("payload"))
payload = data["payload"]
if payload["cmd"] == "sys.get" and payload[
"status"] == "FINISH" and payload["return"] != "":
if payload["args"][0] != "help":
fid = payload["return"]
if "local_path" in payload[
"kwargs"] and "remote_path" in payload["kwargs"]:
local_path = payload["kwargs"]["local_path"]
remote_path = payload["kwargs"]["remote_path"]
else:
local_path = payload["args"][1]
remote_path = payload["args"][0]
stat = payload["kwargs"].get("stat")
if local_path.endswith('/') or os.path.isdir(local_path):
local_path = os.path.join(
local_path, os.path.basename(remote_path))
if checksum(local_path) != fid:
if not check_cache(app_abs_path(self.main_conf.cache),
fid):
FsClient = load_fclient(app_abs_path(
self.main_conf.fs_plugin),
ftype=self.fs_conf.fs_type)
fscli = FsClient(self.fs_conf)
fscli.download(
fid,
os.path.join(
app_abs_path(self.main_conf.cache), fid))
if check_cache(app_abs_path(self.main_conf.cache),
fid):
if not make_dirs(os.path.dirname(local_path)):
log.error("创建目标目录:%s失败" % local_path)
if cp(
os.path.join(
app_abs_path(self.main_conf.cache),
fid), local_path, stat):
payload["return"] = local_path
else:
payload["return"] = ""
else:
payload["return"] = local_path
ret = data
except (ZKClientError, KeyboardInterrupt), e:
log.error(e.message)