class UpYunClient(object):
'''
'''
def __init__(self):
self.ftp_raw_info=open("upyun.txt").read().split(';')
self.bucket_name=self.ftp_raw_info[0]
self.server_user=self.ftp_raw_info[1]
self.server_user_pwd = self.ftp_raw_info[2]
self.base_access_url = self.ftp_raw_info[3]
self.server = UpYun(self.bucket_name,self.server_user,self.server_user_pwd)
self.server.setApiDomain('v0.api.upyun.com')
def UpLoad(self,fileName):
data = open(fileName,'rb')
self.server.setContentMD5(md5file(data))
date_info=datetime.datetime.today()
remote_path=str(date_info.year)+'/'+str(date_info.month)
remote_file_name = '/'+str(date_info.microsecond)+fileName.split('\\')[-1]
to_write= remote_path+remote_file_name
a = self.server.writeFile(to_write,data)
if(a):
return self.base_access_url+'/'+remote_path+remote_file_name
else:
return None
def test_a(request):
from checkpub.models import CheckPub
pdb.set_trace()
conn = UpYun('cpp19', 'one9', 'one9in1234')
cp = CheckPub.objects.get(id=273344)
cp_id = cp.id
#user_id = cp.user.id
op = os.path.abspath(os.path.curdir).split()
#get current img path
userp = cp.photo.url.split()
op.extend(userp)
img_path = ''.join(op)
#open img
data = open('%s' % img_path, 'rb')
#设置待上传文件的 Content-MD5 值
#如又拍云服务端收到的文件MD5值与用户设置的不一致，将回报 406 Not Acceptable 错误
conn.setContentMD5(md5file(data))
#img_format = cp.photo.url.rsplit(".")[-1]
#user_img_jpg = '%d.%s'%(user_id,img_format)
#user_img_jpg = '%d'%(user_id)
result = conn.writeFile('%d' % cp_id, data)
print result
return HttpResponse("success")
def upload_pic(pic, code, up_type='ava'):
#pdb.set_trace()
try:
ext_name = ''
if up_type == 'ava':
conn = UpYun('one9inava', 'one9', 'one9in1234')
elif up_type == 'vid':
conn = UpYun('vid19', 'one9', 'one9in1234')
ext_name = '_cpp.mp4'
elif up_type == 'avd':
conn = UpYun('vid19', 'one9', 'one9in1234')
ext_name = '_ava.mp4'
else:
conn = UpYun('cpp19', 'one9', 'one9in1234')
if 'http' in pic.name:
data = ContentFile(urllib2.urlopen(pic.name).read())
data.seek(0)
elif 'default' in pic.name:
return True
else:
try:
pic.seek(0)
data = pic
except:
print 'no pic'
return False
#conn.setContentMD5(md5file(pic))
#result = conn.writeFile('%d' % code, pic)
#data.write(pic.read())
#data.seek(0)
conn.setContentMD5(md5file(data))
#result = conn.deleteFile('%d' % code)
result = conn.writeFile('%s%s' % (code, ext_name), data)
print result
data.close()
if result:
print 'ok!!'
return True
else:
print 'nok'
return False
except Exception, ex:
print Exception,":",ex
return False
def upload_pic(pic, code, up_type='ava'):
#pdb.set_trace()
try:
ext_name = ''
if up_type == 'ava':
conn = UpYun('one9inava', 'one9', 'one9in1234')
elif up_type == 'vid':
conn = UpYun('vid19', 'one9', 'one9in1234')
ext_name = '_cpp.mp4'
elif up_type == 'avd':
conn = UpYun('vid19', 'one9', 'one9in1234')
ext_name = '_ava.mp4'
else:
conn = UpYun('cpp19', 'one9', 'one9in1234')
if 'http' in pic.name:
data = ContentFile(urllib2.urlopen(pic.name).read())
data.seek(0)
elif 'default' in pic.name:
return True
else:
try:
pic.seek(0)
data = pic
except:
print 'no pic'
return False
#conn.setContentMD5(md5file(pic))
#result = conn.writeFile('%d' % code, pic)
#data.write(pic.read())
#data.seek(0)
conn.setContentMD5(md5file(data))
#result = conn.deleteFile('%d' % code)
result = conn.writeFile('%s%s' % (code, ext_name), data)
print result
data.close()
if result:
print 'ok!!'
return True
else:
print 'nok'
return False
except Exception, ex:
print Exception, ":", ex
return False
def test_dd(request):
u = UpYun('newavatar', 'one9', 'one9in1234')
#查看版本信息
#print u.version()
#设定api所调用的域名
#u.setApiDomain('v0.api.upyun.com')
#创建目录
#a = u.mkDir('/testa')
#print a
#a = u.mkDir('/a/b/c', True) 可自动创建父级目录（最多10级）
#显示目录下的文件
#a = u.readDir('/test/')
#for i in a:
# print i.filename
#开启调试
#u.debug = True
#get user
pdb.set_trace()
user = User.objects.get(id=301)
user_id = user.id
pro = user.get_profile()
#get user avatar url
url_avatar = pro.avatar.url
dir_path = url_avatar.split("/")
del dir_path[-1]
#get upload file path
file_path = "/".join(dir_path)
#create dir
a = u.mkDir('%s' % file_path, True)
#get current dir path
op = os.path.abspath(os.path.curdir).split()
#get current img path
userp = url_avatar.split()
op.extend(userp)
img_path = ''.join(op)
#open img
data = open('%s' % img_path, 'rb')
#设置待上传文件的 Content-MD5 值
#如又拍云服务端收到的文件MD5值与用户设置的不一致，将回报 406 Not Acceptable 错误
u.setContentMD5(md5file(data))
#置待上传文件的 访问密钥（注意：仅支持图片空！，设置密钥后，无法根据原文件URL直接访问，需带 URL 后面加上 （缩略图间隔标志符+密钥） 进行访问）
#如缩略图间隔标志符为 ! ，密钥为 bac，上传文件路径为 /folder/test.jpg ，那么该图片的对外访问地址为： http://空间域名/folder/test.jpg!bac
#u.setFileSecret('bbbb')
#开始上传文件
pdb.set_trace()
#get upload img path
file_path = file_path.split()
img_format = url_avatar.rsplit(".")[-1]
user_img_jpg = '%s.%s' % (user_id, img_format)
file_path.append(user_img_jpg)
upload_path = '/'.join(file_path)
#upload img
a = u.writeFile('%s' % upload_path, data)
print a
#a = u.writeFile('/a/b/c/d/e/f/g/logo.jpg',data, True) 可自动创建父级目录（最多10级）
#spice upyun current img url
domin_url = DOMAIN_NAME.split()
domin_url.append(upload_path)
upyun_img_url = ''.join(domin_url)
#
if a:
pro.avatar = upyun_img_url
pro.save()
#获取上传后的图片信息（仅图片空间有返回数据）
print(u.getWritedFileInfo('x-upyun-width')) # 图片宽度
print(u.getWritedFileInfo('x-upyun-height')) # 图片高度
print(u.getWritedFileInfo('x-upyun-frames')) # 图片帧数
print(u.getWritedFileInfo('x-upyun-file-type')) # 图片类型
#获取文件信息
print u.getFileInfo('/logo.jpg')
#a = u.writeFile('/testd.jpg','sdfsdf')
#print a
#a = u.deleteFile('/testd.jpg')
#print a
a = u.readDir('/')
if a:
for i in a:
print i.filename
else:
print a
class Build(object):
'''
class docs
'''
def __init__(self, space_name, action_type, input_data_dir,
output_data_dir, server_data_dir, action):
'''
init docs
'''
# Init.
self.space_name = space_name
self.input_data_dir = input_data_dir
self.output_data_dir = output_data_dir
self.server_data_dir = server_data_dir
self.action = action
if not os.path.exists(DATA_ORIGIN_DIR):
create_directory(DATA_ORIGIN_DIR)
self.data_origin_file = os.path.join(DATA_ORIGIN_DIR,
"%s.tar.gz" % self.space_name)
if action_type == BUILD_ORIGIN_DATA:
self.build_origin_data(self.action[1])
if action_type == BUILD_UPDATE_PATCH:
self.build_update_patch(self.action[1])
def check_permission(self, space_name):
# Set upyun information.
username = raw_input("Username: "******"/") <> None:
return True
else:
print "验证失败..."
return False
def upload_file(self, local_file, remote_file, data_name):
if self.server.getFileInfo(remote_file):
print "%s: 删除远端%s..." % (self.space_name, data_name)
self.server.deleteFile(remote_file)
print "%s: 删除远端%s完成" % (self.space_name, data_name)
print "%s: 上传远端%s..." % (self.space_name, data_name)
self.server.setContentMD5(md5_file(local_file))
with open(local_file, "rb") as patch_list_file:
if self.server.writeFile(remote_file, patch_list_file.read(),
True):
print "%s: 上传远端%s成功" % (self.space_name, data_name)
else:
print "%s: 上传远端%s失败" % (self.space_name, data_name)
def delete_remote_file(self, remote_file, data_name):
if self.server.getFileInfo(remote_file):
print "%s: 删除远端%s..." % (self.space_name, data_name)
self.server.deleteFile(remote_file)
print "%s: 删除远端%s完成" % (self.space_name, data_name)
else:
print "%s: 远端不存在%s" % (self.space_name, data_name)
def build_origin_data(self, action):
if action == "build":
# Delete origin data
remove_file(self.data_origin_file)
# Build origin data.
print "%s: 创建本地原始数据..." % (self.space_name)
with tarfile.open(self.data_origin_file, "w:gz") as tar:
for root, dir, files in os.walk(self.input_data_dir):
for file in files:
fullpath = os.path.join(root, file)
tar.add(fullpath,
fullpath.split(self.input_data_dir)[1], False)
print "%s: 创建本地原始数据完成" % (self.space_name)
with open(TIME_FLAG_FILE, 'w') as fp:
s = get_current_time("%Y_%m_%d_%H:%M:%S")
fp.write(s)
if action == "upload" and self.check_permission(self.space_name):
remote_origin_file_path = os.path.join(
self.server_data_dir, "origin",
md5_file(self.data_origin_file), "%s.tar.gz" % self.space_name)
self.upload_file(self.data_origin_file, remote_origin_file_path,
"原始数据")
def build_update_patch(self, action):
if os.path.exists(self.data_origin_file):
self.output_patch_dir = os.path.join(self.output_data_dir, "patch")
if not os.path.exists(self.output_patch_dir):
create_directory(self.output_patch_dir)
self.output_temp_dir = os.path.join(self.output_data_dir, "temp")
self.output_temp_file = os.path.join(self.output_temp_dir,
"%s.tar.gz" % self.space_name)
self.output_temp_patch_file = os.path.join(self.output_temp_dir,
"patch")
self.patch_md5_file = os.path.join(self.output_patch_dir,
"patch_md5.json")
self.origin_data_md5 = md5_file(self.data_origin_file)
if not os.path.exists(self.patch_md5_file):
self.patch_md5_json = {}
self.patch_md5_json["origin_data"] = self.origin_data_md5
self.patch_md5_json["current_patch"] = []
else:
self.patch_md5_json = json.load(open(self.patch_md5_file))
if self.patch_md5_json["origin_data"] != self.origin_data_md5:
self.patch_md5_json["origin_data"] = self.origin_data_md5
self.patch_md5_json["current_patch"] = []
self.remote_patch_dir = os.path.join(self.server_data_dir, "patch")
if action == "build":
# Delete temp directory first.
create_directory(self.output_temp_dir, True)
# Build temp file.
print "%s: 创建本地更新数据..." % self.space_name
with tarfile.open(self.output_temp_file, "w:gz") as tar:
for root, dir, files in os.walk(self.input_data_dir):
for file in files:
fullpath = os.path.join(root, file)
tar.add(fullpath,
fullpath.split(self.input_data_dir)[1],
False)
print "%s: 创建本地更新数据完成" % self.space_name
print "%s: 生成补丁文件..." % self.space_name
subprocess.Popen("xdelta3 -ves %s %s %s" %
(self.data_origin_file, self.output_temp_file,
self.output_temp_patch_file),
shell=True).wait()
newest_patch_file_md5 = md5_file(self.output_temp_patch_file)
current_patch_dict = self.patch_md5_json.get("current_patch")
if current_patch_dict:
last_patch_md5 = current_patch_dict[0]["md5"]
if last_patch_md5 == newest_patch_file_md5:
remove_directory(self.output_temp_dir)
print "%s: input_data数据未做任何改变，删除相同补丁文件" % self.space_name
sys.exit(0)
else:
current_patch_dict = []
newest_patch_dir = os.path.join(self.output_patch_dir,
self.origin_data_md5)
if not os.path.exists(newest_patch_dir):
create_directory(newest_patch_dir)
newest_patch_name = "%s-%s.xd3" % (
self.space_name, get_current_time("%Y_%m_%d_%H:%M:%S"))
newest_patch_file = os.path.join(newest_patch_dir,
newest_patch_name)
os.renames(self.output_temp_patch_file, newest_patch_file)
remove_directory(self.output_temp_dir)
current_patch_dict.insert(0, {
"name": newest_patch_name,
"md5": newest_patch_file_md5
})
print "%s: 生成补丁文件完成" % self.space_name
print "%s: 写入补丁md5..." % self.space_name
self.patch_md5_json["current_patch"] = current_patch_dict
with open(self.patch_md5_file, "w") as fp:
json.dump(self.patch_md5_json, fp)
print "%s: 写入补丁md5完成" % self.space_name
elif action == "upload" and self.check_permission(self.space_name):
# Upload patch file.
current_patch_dict = self.patch_md5_json.get("current_patch")
if current_patch_dict != []:
if len(current_patch_dict) > 2:
print "%s: 清理多余的补丁" % self.space_name
spare_patchs = current_patch_dict[2:]
current_patch_dict = current_patch_dict[:2]
for patch in spare_patchs:
patch_name = patch["name"].encode("utf-8")
local_path = os.path.join(self.output_patch_dir,
self.origin_data_md5,
patch_name)
try:
remove_file(local_path)
print "%s: 清除了补丁%s" % (self.space_name,
patch_name)
except:
pass
remote_path = os.path.join(self.remote_patch_dir,
self.origin_data_md5,
patch_name)
self.delete_remote_file(remote_path, patch_name)
self.patch_md5_json[
"current_patch"] = current_patch_dict
with open(self.patch_md5_file, "w") as fp:
json.dump(self.patch_md5_json, fp)
newest_patch_name = current_patch_dict[0]["name"].encode(
"utf-8")
newest_patch_file = os.path.join(self.output_patch_dir,
self.origin_data_md5,
newest_patch_name)
remote_patch_file = os.path.join(self.remote_patch_dir,
self.origin_data_md5,
newest_patch_name)
remote_patch_md5_file = os.path.join(
self.remote_patch_dir, self.origin_data_md5,
"patch_md5.json")
# upload newest_patch_file
self.upload_file(newest_patch_file, remote_patch_file,
"补丁更新数据")
# Update patch list file.
self.upload_file(self.patch_md5_file,
remote_patch_md5_file, "补丁md5列表文件")
else:
print "%s: 当前没有任何补丁，请打好补丁再上传吧！" % self.space_name
else:
print "%s: %s 不存在， 无法进行补丁的创建和上传" % (self.space_name,
self.data_origin_file)
#a = u.mkDir('/testa')
#print a
#a = u.mkDir('/a/b/c', True) 可自动创建父级目录（最多10级）
#显示目录下的文件
#a = u.readDir('/test/')
#for i in a:
# print i.filename
#开启调试
#u.debug = True
data = open('aa.gif','rb')
#设置待上传文件的 Content-MD5 值
#如又拍云服务端收到的文件MD5值与用户设置的不一致，将回报 406 Not Acceptable 错误
u.setContentMD5(md5file(data))
#置待上传文件的 访问密钥（注意：仅支持图片空！，设置密钥后，无法根据原文件URL直接访问，需带 URL 后面加上 （缩略图间隔标志符+密钥） 进行访问）
#如缩略图间隔标志符为 ! ，密钥为 bac，上传文件路径为 /folder/test.jpg ，那么该图片的对外访问地址为： http://空间域名/folder/test.jpg!bac
u.setFileSecret('bbbb')
#开始上传文件
a = u.writeFile('/logo.jpg',data)
print a
#a = u.writeFile('/a/b/c/d/e/f/g/logo.jpg',data, True) 可自动创建父级目录（最多10级）
#获取上传后的图片信息（仅图片空间有返回数据）
print(u.getWritedFileInfo('x-upyun-width')) # 图片宽度
print(u.getWritedFileInfo('x-upyun-height')) # 图片高度
print(u.getWritedFileInfo('x-upyun-frames')) # 图片帧数
print(u.getWritedFileInfo('x-upyun-file-type')) # 图片类型
class Build(object):
'''
class docs
'''
def __init__(self,
space_name,
action_type,
input_data_dir,
output_data_dir,
server_data_dir,
action):
'''
init docs
'''
# Init.
self.space_name = space_name
self.input_data_dir = input_data_dir
self.output_data_dir = output_data_dir
self.server_data_dir = server_data_dir
self.action = action
if not os.path.exists(DATA_ORIGIN_DIR):
create_directory(DATA_ORIGIN_DIR)
self.data_origin_file = os.path.join(DATA_ORIGIN_DIR, "%s.tar.gz" % self.space_name)
if action_type == BUILD_ORIGIN_DATA:
self.build_origin_data(self.action[1])
if action_type == BUILD_UPDATE_PATCH:
self.build_update_patch(self.action[1])
def check_permission(self, space_name):
# Set upyun information.
username = raw_input("Username: "******"/") <> None:
return True
else:
print "验证失败..."
return False
def upload_file(self,
local_file,
remote_file,
data_name):
if self.server.getFileInfo(remote_file):
print "%s: 删除远端%s..." % (self.space_name, data_name)
self.server.deleteFile(remote_file)
print "%s: 删除远端%s完成" % (self.space_name, data_name)
print "%s: 上传远端%s..." % (self.space_name, data_name)
self.server.setContentMD5(md5_file(local_file))
with open(local_file, "rb") as patch_list_file:
if self.server.writeFile(remote_file, patch_list_file.read(), True):
print "%s: 上传远端%s成功" % (self.space_name, data_name)
else:
print "%s: 上传远端%s失败" % (self.space_name, data_name)
def delete_remote_file(self, remote_file, data_name):
if self.server.getFileInfo(remote_file):
print "%s: 删除远端%s..." % (self.space_name, data_name)
self.server.deleteFile(remote_file)
print "%s: 删除远端%s完成" % (self.space_name, data_name)
else:
print "%s: 远端不存在%s" % (self.space_name, data_name)
def build_origin_data(self, action):
if action == "build":
# Delete origin data
remove_file(self.data_origin_file)
# Build origin data.
print "%s: 创建本地原始数据..." % (self.space_name)
with tarfile.open(self.data_origin_file, "w:gz") as tar:
for root, dir, files in os.walk(self.input_data_dir):
for file in files:
fullpath=os.path.join(root, file)
tar.add(fullpath, fullpath.split(self.input_data_dir)[1], False)
print "%s: 创建本地原始数据完成" % (self.space_name)
with open(TIME_FLAG_FILE, 'w') as fp:
s = get_current_time("%Y_%m_%d_%H:%M:%S")
fp.write(s)
if action == "upload" and self.check_permission(self.space_name):
remote_origin_file_path = os.path.join(
self.server_data_dir,
"origin",
md5_file(self.data_origin_file),
"%s.tar.gz" % self.space_name)
self.upload_file(self.data_origin_file, remote_origin_file_path, "原始数据")
def build_update_patch(self, action):
if os.path.exists(self.data_origin_file):
self.output_patch_dir = os.path.join(self.output_data_dir, "patch")
if not os.path.exists(self.output_patch_dir):
create_directory(self.output_patch_dir)
self.output_temp_dir = os.path.join(self.output_data_dir, "temp")
self.output_temp_file = os.path.join(self.output_temp_dir, "%s.tar.gz" % self.space_name)
self.output_temp_patch_file = os.path.join(self.output_temp_dir, "patch")
self.patch_md5_file = os.path.join(self.output_patch_dir, "patch_md5.json")
self.origin_data_md5 = md5_file(self.data_origin_file)
if not os.path.exists(self.patch_md5_file):
self.patch_md5_json = {}
self.patch_md5_json["origin_data"] = self.origin_data_md5
self.patch_md5_json["current_patch"] = []
else:
self.patch_md5_json = json.load(open(self.patch_md5_file))
if self.patch_md5_json["origin_data"] != self.origin_data_md5:
self.patch_md5_json["origin_data"] = self.origin_data_md5
self.patch_md5_json["current_patch"] = []
self.remote_patch_dir = os.path.join(self.server_data_dir, "patch")
if action == "build":
# Delete temp directory first.
create_directory(self.output_temp_dir, True)
# Build temp file.
print "%s: 创建本地更新数据..." % self.space_name
with tarfile.open(self.output_temp_file, "w:gz") as tar:
for root, dir, files in os.walk(self.input_data_dir):
for file in files:
fullpath=os.path.join(root, file)
tar.add(fullpath, fullpath.split(self.input_data_dir)[1], False)
print "%s: 创建本地更新数据完成" % self.space_name
print "%s: 生成补丁文件..." % self.space_name
subprocess.Popen(
"xdelta3 -ves %s %s %s" % (
self.data_origin_file,
self.output_temp_file,
self.output_temp_patch_file),
shell=True).wait()
newest_patch_file_md5 = md5_file(self.output_temp_patch_file)
current_patch_dict = self.patch_md5_json.get("current_patch")
if current_patch_dict:
last_patch_md5 = current_patch_dict[0]["md5"]
if last_patch_md5 == newest_patch_file_md5:
remove_directory(self.output_temp_dir)
print "%s: input_data数据未做任何改变，删除相同补丁文件" % self.space_name
sys.exit(0)
else:
current_patch_dict = []
newest_patch_dir = os.path.join(self.output_patch_dir, self.origin_data_md5)
if not os.path.exists(newest_patch_dir):
create_directory(newest_patch_dir)
newest_patch_name = "%s-%s.xd3" % (self.space_name, get_current_time("%Y_%m_%d_%H:%M:%S"))
newest_patch_file = os.path.join(newest_patch_dir, newest_patch_name)
os.renames(self.output_temp_patch_file, newest_patch_file)
remove_directory(self.output_temp_dir)
current_patch_dict.insert(0, {"name" : newest_patch_name, "md5" : newest_patch_file_md5})
print "%s: 生成补丁文件完成" % self.space_name
print "%s: 写入补丁md5..." % self.space_name
self.patch_md5_json["current_patch"] = current_patch_dict
with open(self.patch_md5_file, "w") as fp:
json.dump(self.patch_md5_json, fp)
print "%s: 写入补丁md5完成" % self.space_name
elif action == "upload" and self.check_permission(self.space_name):
# Upload patch file.
current_patch_dict = self.patch_md5_json.get("current_patch")
if current_patch_dict != []:
if len(current_patch_dict) > 2:
print "%s: 清理多余的补丁" % self.space_name
spare_patchs = current_patch_dict[2:]
current_patch_dict = current_patch_dict[:2]
for patch in spare_patchs:
patch_name = patch["name"].encode("utf-8")
local_path = os.path.join(self.output_patch_dir, self.origin_data_md5, patch_name)
try:
remove_file(local_path)
print "%s: 清除了补丁%s" % (self.space_name, patch_name)
except:
pass
remote_path = os.path.join(self.remote_patch_dir, self.origin_data_md5, patch_name)
self.delete_remote_file(remote_path, patch_name)
self.patch_md5_json["current_patch"] = current_patch_dict
with open(self.patch_md5_file, "w") as fp:
json.dump(self.patch_md5_json, fp)
newest_patch_name = current_patch_dict[0]["name"].encode("utf-8")
newest_patch_file = os.path.join(self.output_patch_dir, self.origin_data_md5, newest_patch_name)
remote_patch_file = os.path.join(self.remote_patch_dir, self.origin_data_md5, newest_patch_name)
remote_patch_md5_file = os.path.join(self.remote_patch_dir, self.origin_data_md5, "patch_md5.json")
# upload newest_patch_file
self.upload_file(newest_patch_file, remote_patch_file, "补丁更新数据")
# Update patch list file.
self.upload_file(self.patch_md5_file, remote_patch_md5_file, "补丁md5列表文件")
else:
print "%s: 当前没有任何补丁，请打好补丁再上传吧！" % self.space_name
else:
print "%s: %s 不存在， 无法进行补丁的创建和上传" % (self.space_name, self.data_origin_file)
