def syncLocalToCOS(appid,
secret_id,
secret_key,
bucket,
region_info,
root,
subFolder='',
debug=1):
root = formatPath(root)
subFolder = formatPath(subFolder)
cos_client = CosClient(appid, secret_id, secret_key, region=region_info)
# check cos_client & bucket
list_folder_ret = cos_client.list_folder(ListFolderRequest(bucket, '/'))
if list_folder_ret['message'] != 'SUCCESS':
raise Exception(
'>>> Check your appid / secret_id / secret_key / bucket_name <<<')
# 读取本地需要更新的目录
localFilesDict, localEmptyFolders = readLocalFiles(root,
subFolder,
debug=debug)
# 读取cos上需要更新的目录
cosFilesDict, cosEmptyFolders = readCosFiles(cos_client,
bucket,
subFolder,
debug=debug)
# 比较文件修改时间，筛选出需要更新的文件，并且上传。
modifiedLocalFiles = filterModifiedLocalFiles(localFilesDict,
cosFilesDict,
debug=debug)
if modifiedLocalFiles:
start = datetime.now()
if debug: drawTitle('Uploading files')
for file in modifiedLocalFiles:
uploadToCos(cos_client, bucket, root, file)
if debug: print('---\n%s: %s' % ('Used', datetime.now() - start))
# 筛选出cos上有，但本地已经不存在的文件，删除COS上的文件。
extraCosFiles = filterExtraCosFiles(localFilesDict,
cosFilesDict,
debug=debug)
if extraCosFiles:
start = datetime.now()
if debug: drawTitle('Deleting COS files')
for file in extraCosFiles:
deleteCosFile(cos_client, bucket, file)
if debug: print('---\n%s: %s' % ('Used', datetime.now() - start))
# 同步空文件夹
syncEmptyFolders(cos_client,
bucket,
localEmptyFolders,
cosEmptyFolders,
debug=debug)
class CosFS(object):
def __init__(self, appid, secret_id, secret_key, bucket, region = None):
self.appid = appid
self.secret_id = secret_id
self.secret_key = secret_key
self.bucket = bucket
if region:
self.cos_client = CosClient(appid, secret_id, secret_key, region=region)
else:
self.cos_client = CosClient(appid, secret_id, secret_key)
def list_dir(self, path=u'/'):
path = to_unicode(path)
prefix = u''
if path.endswith(u'*'):
prefix = os.path.basename(path)[:-1]
path = os.path.dirname(path)
if not path.endswith(u'/'):
path += u'/'
context=u''
arr_data = []
while True:
request = ListFolderRequest(self.bucket, path, prefix=prefix, context=context)
result = self.cos_client.list_folder(request)
if result['code'] != 0:
raise CosFSException(result['code'], result['message'] + ': ' + path)
data = result['data']
arr_data.append(data)
has_more = False
if 'has_more' in data:
has_more = data['has_more']
if 'listover' in data: #cos v4
has_more = not data['listover']
if has_more:
context = data['context']
else:
break
dataset = {'dircount': 0, 'filecount': 0, 'infos': []}
for data in arr_data:
if 'filecount' not in data:
data['filecount'] = 0
data['dircount'] = 0
for entry in data['infos']:
if self.isFile(entry):
data['filecount'] += 1
else:
entry['name'] = entry['name'].rstrip(u'/')
data['dircount'] += 1
dataset['dircount'] += data['dircount']
dataset['filecount'] += data['filecount']
dataset['infos'] += data['infos']
return dataset
def ls(self, path=u'/', detail=False, recursive=False):
content = self.list_dir(path)
if recursive:
print '%s:' % (path.encode('utf-8'))
for entry in content['infos']:
isFile = self.isFile(entry)
name = entry['name'].encode('utf-8')
if detail:
ctime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(entry['ctime'])))
print './%s%s: [size:%d] [created_at:%s]' % (name, '' if isFile else '/', entry['filesize'] if isFile else 0, ctime)
else:
print name + ('' if isFile else '/')
if recursive and content['dircount'] > 0:
print ''
for info in content['infos']:
if 'sha' not in info:
self.ls(path.rstrip(u'/') + u'/' + info['name'], detail, recursive)
print ''
def mv(self, src, dest, overwrite=False):
src = to_unicode(src)
dest = to_unicode(dest)
if dest.endswith(u'/'):
dest += os.path.basename(src)
request = MoveFileRequest(self.bucket, src, dest)
result = self.cos_client.move_file(request)
if result['code'] != 0:
raise CosFSException(result['code'], result['message'])
def download(self, remote, local, overwrite=False):
local = to_unicode(local)
remote = to_unicode(remote)
if local == u'.':
local += u'/'
if local.endswith(u'/'):
local += os.path.basename(remote)
if os.path.exists(local):
if overwrite:
os.unlink(local)
else:
raise CosFSException(-1, 'local file %s exists' % local)
fileattr = self.stat(remote)
url = fileattr['source_url'] + '?sign=' + fileattr['sign']
download_file(url, local)
def cat(self, path):
fileattr = self.stat(to_unicode(path))
url = fileattr['source_url'] + '?sign=' + fileattr['sign']
download_file(url, '/dev/stdout')
def upload(self, local, remote, overwrite=False, silent=False):
local = to_unicode(local)
remote = to_unicode(remote)
if remote.endswith(u'/'):
remote += os.path.basename(local)
request = UploadFileRequest(self.bucket, remote, local)
if overwrite:
request.set_insert_only(0)
result = self.cos_client.upload_file(request)
if result['code'] != 0:
if result['code'] == CODE_SAME_FILE:
if not silent:
print >>sys.stderr, "skipped: same file on COS"
return
if overwrite and result['code'] == CODE_ERR_OFF_GOBACK:
print >>sys.stderr, "fix tencent bug: remove remote file when ErrOffGoBack occurs"
"""
上传出现这个4024的错误之后，覆盖（insertonly=0）参数也不能成功，只能删除文件后重新上传
建议您将分片大小改为1M 分片上传之间sleep 100ms 出现错误的概率会小很多
抱歉，这个问题暂时无法彻底解决，给您带来了不便。
"""
self.rm(remote)
raise CosFSException(result['code'], result['message'])
def cp(self, src, dest, overwrite=False):
if src.startswith('cos:') and dest.startswith('cos:'): #move in cos
raise CosFSException(-1, "not supported yet, use mv please")
elif src.startswith('cos:'): #download
self.download(src[4:], dest, overwrite)
elif dest.startswith('cos:'): #upload
self.upload(src, dest[4:], overwrite)
else:
raise CosFSException(-1, "at least one of src/dest should start with `cos:`")
def cpdir(self, src, dest, conflict=CONFLICT_ERROR):
begin_at = time.time()
src = to_unicode(src)
dest = to_unicode(dest)
if src.startswith(u'cos:') and dest.startswith(u'cos:'):
raise CosFSException(-1, "not supported")
elif src.startswith(u'cos:'): #download
self.downloadDir(src[4:], dest, conflict)
elif dest.startswith(u'cos:'): #upload
self.uploadDir(src, dest[4:], conflict)
else:
raise CosFSException(-1, "at least one of src/dest should start with `cos:`")
usage = time.time() - begin_at
#print '[Time Usage: %.6f]' % usage
#将[cos上remote目录里]的内容下载到[本地local目录里]，如果local目录不存在，会被创建
def downloadDir(self, remote, local, conflict):
remote = remote.rstrip(u'/')
local = local.rstrip(u'/')
file_queue = Queue.Queue()
def walk(path = u'/', level = 0):
print '[mkdir] ' + ' ' * level + local + path
localMkdir(local + path)
content = self.list_dir(remote + path)
for entry in content['infos']:
name = path + entry['name'].encode('utf-8')
if self.isFile(entry):
print '[copy] ' + ' ' * level + local + name
overwrite = conflict == CONFLICT_OVERWRITE
file_queue.put([self.download, (remote + name, local + name, overwrite)])
else:
walk(name + '/', level + 1)
walk()
CosThread.execute(file_queue)
print >>sys.stderr, "[download finished]"
def uploadDir(self, local, remote, conflict):
if not remote.endswith(u'/'):
remote += u'/'
if not local.endswith(u'/'):
remote += os.path.basename(local) + u'/'
local = os.path.abspath(local) + u'/'
if not os.path.isdir(local):
raise CosFSException(-1, "please specify a local directory")
retry(self.mkdir, remote)
def uploadFile(localfile, remotefile):
try:
print >>sys.stderr, '[uploadFile] %s => %s' % (localfile, remotefile)
self.upload(localfile, remotefile, silent=True)
print >>sys.stderr, 'done: new'
except Exception, e:
if e[0] == CODE_EXISTED: #ERROR_CMD_COS_FILE_EXIST
if conflict == CONFLICT_SKIP:
print >>sys.stderr, 'skipped: existed'
elif conflict == CONFLICT_OVERWRITE:
self.upload(localfile, remotefile, overwrite=True)
print >>sys.stderr, 'done: overwrite'
else:
raise
return
raise
file_queue = Queue.Queue()
def process_dir(arg, dirname, filelist):
dir_suffix = dirname[len(local):]
remote_dir = remote + dir_suffix
file_queue.put([self.mkdir, (remote_dir,)])
print >>sys.stderr, '[doUpload] queue for mkdir %s' % remote_dir
for filename in filelist:
localfile = dirname.rstrip(u'/') + u'/' + filename
if os.path.isdir(localfile):
continue
if os.path.islink(localfile):
print >>sys.stderr, '[doUpload] skip symlink %s' % (localfile)
continue
remotefile = remote_dir.rstrip(u'/') + u'/' + filename
file_queue.put([uploadFile, (localfile, remotefile)])
print >>sys.stderr, '[doUpload] queue for upload file %s ...' % remotefile
os.path.walk(local, process_dir, None)
CosThread.execute(file_queue)
print >>sys.stderr, "[upload finished]"
create_folder_ret = self.cos_client.create_folder(request)
result = json.loads(create_folder_ret)
return result
def DeleteFolderTest(self):
request = DelFolderRequest(self.bucket, self.cos_path)
delete_folder_ret = CosClient.del_folder(request)
result = json.loads(delete_folder_ret)
print 'delete folder ret:', repr(delete_folder_ret)
return result
if __name__ == '__main__':
ar = CosClient(appid=AudioRecord.appid,
secret_id=AudioRecord.secret_id,
secret_key=AudioRecord.secret_key)
request = ListFolderRequest(u'porncheck', u'/1400011479/19700101/')
list_folder_ret = ar.list_folder(request)
print list_folder_ret
#request = DelFolderRequest(u'porncheck', u'/1400011479/19700101/')
#delete_folder_ret = ar.del_folder(request)
#print 'delete folder ret:', repr(delete_folder_ret)
#ar = AudioRecord('heydo','/record/introduce/102/')
#ar = AudioRecord('porncheck', '/1400011479/19700101/')
#print ar
#print ar.get_once_sign()
#print ar.DeleteFolderTest()
#print ar.get_once_sign()
#print ar.CreateIntroduceDir()
#cos_demo()
class CosV4StorageService(storage_service.StorageService):
def __init__(self, *args, **kwargs):
appid = int(kwargs['appid'])
region = kwargs['region']
accesskeyid = unicode(kwargs['accesskeyid'])
accesskeysecret = unicode(kwargs['accesskeysecret'])
bucket = unicode(kwargs['bucket'])
if 'prefix_dir' in kwargs:
self._prefix_dir = kwargs['prefix_dir']
else:
self._prefix_dir = None
self._cos_api = CosClient(appid,
accesskeyid,
accesskeysecret,
region=region)
self._bucket = bucket
self._overwrite = kwargs[
'overwrite'] == 'true' if 'overwrite' in kwargs else False
self._max_retry = 20
def download(self, task, local_path):
# self._oss_api.get_object_to_file(urllib.unquote(cos_path).encode('utf-8'), local_path)
for i in range(20):
logger.info("download file with rety {0}".format(i))
import os
try:
os.remove(task.key)
except:
pass
req = DownloadFileRequest(self._bucket, task.key, local_path)
ret = self._cos_api.download_file(req)
# self._oss_api.get_object_to_file(task.key, local_path)
logger.debug(str(ret))
if task.size is None:
logger.info(
"task's size is None, skip check file size on local")
break
from os import path
if path.getsize(local_path) != int(task.size):
logger.error(
"Download Failed, size1: {size1}, size2: {size2}".format(
size1=path.getsize(local_path), size2=task.size))
else:
logger.info("Download Successfully, break")
break
else:
raise IOError("Download Failed with 20 retry")
def upload(self, task, local_path):
cos_path = task.key
if not cos_path.startswith('/'):
cos_path = '/' + cos_path
if self._prefix_dir:
cos_path = self._prefix_dir + cos_path
if isinstance(local_path, unicode):
local_path.encode('utf-8')
insert_only = 0 if self._overwrite else 1
for i in range(10):
try:
upload_request = UploadFileRequest(self._bucket,
unicode(cos_path),
local_path,
insert_only=insert_only)
upload_file_ret = self._cos_api.upload_file(upload_request)
if upload_file_ret[u'code'] != 0:
logger.warn("upload failed:" + str(upload_file_ret))
raise OSError("UploadError: " + str(upload_file_ret))
else:
break
except:
pass
else:
raise IOError("upload failed")
def list(self):
if self._prefix_dir is None:
for i in self.dfs('/'):
yield i
else:
for i in self.dfs(self._prefix_dir):
yield i
def dfs(self, path):
print "DFS: {path}".format(path=to_utf8(path))
_finish = False
_context = u''
max_retry = self._max_retry
path = to_unicode(path)
while not _finish:
request = ListFolderRequest(bucket_name=self._bucket,
cos_path=path,
context=_context)
ret = self._cos_api.list_folder(request)
if ret['code'] != 0:
max_retry -= 1
else:
_finish = ret['data']['listover']
_context = ret['data']['context']
for item in ret['data']['infos']:
if 'filelen' in item:
try:
key = "{prefix}{filename}".format(
prefix=path, filename=item['name'])
yield Task(key, item['filelen'], None)
except:
pass
else:
_sub_dir = "{prefix}{filename}".format(
prefix=path.encode('utf-8'),
filename=item['name'].encode('utf-8'))
for i in self.dfs(_sub_dir):
yield i
if max_retry == 0:
_finish = True
def exists(self, task):
_path = task.key
_size = task.size
if not _path.startswith('/'):
_path = '/' + _path
logger = getLogger(__name__)
# logger.info("func: exists: " + str(_path))
if self._prefix_dir:
_path = self._prefix_dir + _path
if isinstance(_path, str):
_path = _path.decode('utf-8')
request = StatFileRequest(self._bucket, _path)
ret = self._cos_api.stat_file(request)
logger.debug("ret: " + str(ret))
# import json
# v = json.loads(ret)
if ret['code'] != 0:
# logger.warn("error code: " + str(ret['code']))
return False
if ret['data']['filelen'] != ret['data']['filesize']:
logger.warn(
"file is broken, filelen: {len}, filesize: {size}".format(
len=ret['data']['filelen'], size=ret['data']['filesize']))
return False
elif _size is not None and ret['data']['filelen'] != _size:
return False
elif ret['data']['filelen'] != _size:
return False
return True
def cos_demo():
# 设置用户属性, 包括appid, secret_id和secret_key
# 这些属性可以在cos控制台获取(https://console.qcloud.com/cos)
appid = 111 # 替换为用户的appid
secret_id = u'' # 替换为用户的secret_id
secret_key = u'' # 替换为用户的secret_key
region = "shanghai" # # 替换为用户的region，目前可以为 shanghai/guangzhou
cos_client = CosClient(appid, secret_id, secret_key, region)
# 设置要操作的bucket
bucket = u''
############################################################################
# 文件操作 #
############################################################################
# 1. 上传文件(默认不覆盖)
# 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt
# 默认不覆盖, 如果cos上文件存在,则会返回错误
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_1.txt')
upload_file_ret = cos_client.upload_file(request)
logger.info("upload file, return message: " + str(upload_file_ret))
# 分片上传大文件
request = UploadSliceFileRequest(bucket, u'/sample_bigfile.txt',
u'local_bigfile.txt')
ret = cos_client.upload_slice_file(request)
logger.info("slice upload, return message: " + str(ret))
# 2. 上传文件(覆盖文件)
# 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt
with open('local_file_2.txt', 'w') as f:
f.write("hello world2")
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_2.txt')
request.set_insert_only(0) # 设置允许覆盖
upload_file_ret = cos_client.upload_file(request)
logger.info('overwrite file, return message:' + str(upload_file_ret))
# 3. 获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
logger.info('stat file, return message: ' + str(stat_file_ret))
# 4. 更新文件属性
request = UpdateFileRequest(bucket, u'/sample_file.txt')
request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性
request.set_authority(u'eWRPrivate') # 设置文件的权限
request.set_cache_control(u'cache_xxx') # 设置Cache-Control
request.set_content_type(u'application/text') # 设置Content-Type
request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition
request.set_content_language(u'english') # 设置Content-Language
request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性
request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性
update_file_ret = cos_client.update_file(request)
logger.info('update file, return message: ' + str(update_file_ret))
# 5. 更新后再次获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
logger.info('stat file, return message: ' + str(stat_file_ret))
############################################################################
# 目录操作 #
############################################################################
# 1. 生成目录, 目录名为sample_folder
request = CreateFolderRequest(bucket, u'/sample_folder/')
create_folder_ret = cos_client.create_folder(request)
logger.info('create folder ret:' + str(create_folder_ret))
# 2. 更新目录的biz_attr属性
request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录')
update_folder_ret = cos_client.update_folder(request)
logger.info('update folder ret:' + str(update_folder_ret))
# 3. 获取目录属性
request = StatFolderRequest(bucket, u'/sample_folder/')
stat_folder_ret = cos_client.stat_folder(request)
logger.info("stat folder, return message: " + str(stat_folder_ret))
# 4. list目录, 获取目录下的成员
request = ListFolderRequest(bucket, u'/sample_folder/')
list_folder_ret = cos_client.list_folder(request)
logger.info("list folder, return message: " + str(list_folder_ret))
# 5. 删除目录
request = DelFolderRequest(bucket, u'/sample_folder/')
delete_folder_ret = cos_client.del_folder(request)
logger.info("delete folder, return message: " + str(delete_folder_ret))
def cos_demo():
# 设置用户属性, 包括appid, secret_id和secret_key
# 这些属性可以在cos控制台获取(https://console.qcloud.com/cos)
appid = 100000 # 替换为用户的appid
secret_id = u'xxxxxxxx' # 替换为用户的secret_id
secret_key = u'xxxxxxx' # 替换为用户的secret_key
cos_client = CosClient(appid, secret_id, secret_key)
# 设置要操作的bucket
bucket = u'mybucket'
############################################################################
# 文件操作 #
############################################################################
# 1. 上传文件(默认不覆盖)
# 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt
# 默认不覆盖, 如果cos上文件存在,则会返回错误
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_1.txt')
upload_file_ret = cos_client.upload_file(request)
print 'upload file ret:', repr(upload_file_ret)
# 2. 上传文件(覆盖文件)
# 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_2.txt')
request.set_insert_only(0) # 设置允许覆盖
upload_file_ret = cos_client.upload_file(request)
print 'overwrite file ret:', repr(upload_file_ret)
# 3. 获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
print 'stat file ret:', repr(stat_file_ret)
# 4. 更新文件属性
request = UpdateFileRequest(bucket, u'/sample_file.txt')
request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性
request.set_authority(u'eWRPrivate') # 设置文件的权限
request.set_cache_control(u'cache_xxx') # 设置Cache-Control
request.set_content_type(u'application/text') # 设置Content-Type
request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition
request.set_content_language(u'english') # 设置Content-Language
request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性
request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性
update_file_ret = cos_client.update_file(request)
print 'update file ret:', repr(update_file_ret)
# 5. 更新后再次获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
print 'stat file ret:', repr(stat_file_ret)
# 6. 移动文件, 将sample_file.txt移动位sample_file_move.txt
request = MoveFileRequest(bucket, u'/sample_file.txt',
u'/sample_file_move.txt')
stat_file_ret = cos_client.move_file(request)
print 'move file ret:', repr(stat_file_ret)
# 7. 删除文件
request = DelFileRequest(bucket, u'/sample_file_move.txt')
del_ret = cos_client.del_file(request)
print 'del file ret:', repr(del_ret)
############################################################################
# 目录操作 #
############################################################################
# 1. 生成目录, 目录名为sample_folder
request = CreateFolderRequest(bucket, u'/sample_folder/')
create_folder_ret = cos_client.create_folder(request)
print 'create folder ret:', create_folder_ret
# 2. 更新目录的biz_attr属性
request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录')
update_folder_ret = cos_client.update_folder(request)
print 'update folder ret:', repr(update_folder_ret)
# 3. 获取目录属性
request = StatFolderRequest(bucket, u'/sample_folder/')
stat_folder_ret = cos_client.stat_folder(request)
print 'stat folder ret:', repr(stat_folder_ret)
# 4. list目录, 获取目录下的成员
request = ListFolderRequest(bucket, u'/sample_folder/')
list_folder_ret = cos_client.list_folder(request)
print 'list folder ret:', repr(list_folder_ret)
# 5. 删除目录
request = DelFolderRequest(bucket, u'/sample_folder/')
delete_folder_ret = cos_client.del_folder(request)
print 'delete folder ret:', repr(delete_folder_ret)
def cos_demo():
# 设置用户属性, 包括appid, secret_id和secret_key
# 这些属性可以在cos控制台获取(https://console.qcloud.com/cos)
appid = 100000 # 替换为用户的appid
secret_id = u"xxxxxxxx" # 替换为用户的secret_id
secret_key = u"xxxxxxx" # 替换为用户的secret_key
cos_client = CosClient(appid, secret_id, secret_key)
# 设置要操作的bucket
bucket = u"mybucket"
############################################################################
# 文件操作 #
############################################################################
# 1. 上传文件(默认不覆盖)
# 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt
# 默认不覆盖, 如果cos上文件存在,则会返回错误
request = UploadFileRequest(bucket, u"/sample_file.txt", u"local_file_1.txt")
upload_file_ret = cos_client.upload_file(request)
print "upload file ret:", repr(upload_file_ret)
# 2. 上传文件(覆盖文件)
# 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt
request = UploadFileRequest(bucket, u"/sample_file.txt", u"local_file_2.txt")
request.set_insert_only(0) # 设置允许覆盖
upload_file_ret = cos_client.upload_file(request)
print "overwrite file ret:", repr(upload_file_ret)
# 3. 获取文件属性
request = StatFileRequest(bucket, u"/sample_file.txt")
stat_file_ret = cos_client.stat_file(request)
print "stat file ret:", repr(stat_file_ret)
# 4. 更新文件属性
request = UpdateFileRequest(bucket, u"/sample_file.txt")
request.set_biz_attr(u"这是个demo文件") # 设置文件biz_attr属性
request.set_authority(u"eWRPrivate") # 设置文件的权限
request.set_cache_control(u"cache_xxx") # 设置Cache-Control
request.set_content_type(u"application/text") # 设置Content-Type
request.set_content_disposition(u"ccccxxx.txt") # 设置Content-Disposition
request.set_content_language(u"english") # 设置Content-Language
request.set_x_cos_meta(u"x-cos-meta-xxx", u"xxx") # 设置自定义的x-cos-meta-属性
request.set_x_cos_meta(u"x-cos-meta-yyy", u"yyy") # 设置自定义的x-cos-meta-属性
update_file_ret = cos_client.update_file(request)
print "update file ret:", repr(update_file_ret)
# 5. 更新后再次获取文件属性
request = StatFileRequest(bucket, u"/sample_file.txt")
stat_file_ret = cos_client.stat_file(request)
print "stat file ret:", repr(stat_file_ret)
# 6. 移动文件, 将sample_file.txt移动位sample_file_move.txt
request = MoveFileRequest(bucket, u"/sample_file.txt", u"/sample_file_move.txt")
stat_file_ret = cos_client.move_file(request)
print "move file ret:", repr(stat_file_ret)
# 7. 删除文件
request = DelFileRequest(bucket, u"/sample_file_move.txt")
del_ret = cos_client.del_file(request)
print "del file ret:", repr(del_ret)
############################################################################
# 目录操作 #
############################################################################
# 1. 生成目录, 目录名为sample_folder
request = CreateFolderRequest(bucket, u"/sample_folder/")
create_folder_ret = cos_client.create_folder(request)
print "create folder ret:", create_folder_ret
# 2. 更新目录的biz_attr属性
request = UpdateFolderRequest(bucket, u"/sample_folder/", u"这是一个测试目录")
update_folder_ret = cos_client.update_folder(request)
print "update folder ret:", repr(update_folder_ret)
# 3. 获取目录属性
request = StatFolderRequest(bucket, u"/sample_folder/")
stat_folder_ret = cos_client.stat_folder(request)
print "stat folder ret:", repr(stat_folder_ret)
# 4. list目录, 获取目录下的成员
request = ListFolderRequest(bucket, u"/sample_folder/")
list_folder_ret = cos_client.list_folder(request)
print "list folder ret:", repr(list_folder_ret)
# 5. 删除目录
request = DelFolderRequest(bucket, u"/sample_folder/")
delete_folder_ret = cos_client.del_folder(request)
print "delete folder ret:", repr(delete_folder_ret)
def cos_demo():
# 设置用户属性, 包括appid, secret_id和secret_key
# 这些属性可以在cos控制台获取(https://console.qcloud.com/cos)
appid = 111 # 替换为用户的appid
secret_id = u'' # 替换为用户的secret_id
secret_key = u'' # 替换为用户的secret_key
region = "shanghai" # # 替换为用户的region，目前可以为 shanghai/guangzhou
cos_client = CosClient(appid, secret_id, secret_key, region)
# 设置要操作的bucket
bucket = u''
############################################################################
# 文件操作 #
############################################################################
# 1. 上传文件(默认不覆盖)
# 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt
# 默认不覆盖, 如果cos上文件存在,则会返回错误
request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_1.txt')
upload_file_ret = cos_client.upload_file(request)
logger.info("upload file, return message: " + str(upload_file_ret))
# 分片上传大文件
request = UploadSliceFileRequest(bucket, u'/sample_bigfile.txt', u'local_bigfile.txt')
ret = cos_client.upload_slice_file(request)
logger.info("slice upload, return message: " + str(ret))
# 2. 上传文件(覆盖文件)
# 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt
with open('local_file_2.txt', 'w') as f:
f.write("hello world2")
request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_2.txt')
request.set_insert_only(0) # 设置允许覆盖
upload_file_ret = cos_client.upload_file(request)
logger.info('overwrite file, return message:' + str(upload_file_ret))
# 3. 获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
logger.info('stat file, return message: ' + str(stat_file_ret))
# 4. 更新文件属性
request = UpdateFileRequest(bucket, u'/sample_file.txt')
request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性
request.set_authority(u'eWRPrivate') # 设置文件的权限
request.set_cache_control(u'cache_xxx') # 设置Cache-Control
request.set_content_type(u'application/text') # 设置Content-Type
request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition
request.set_content_language(u'english') # 设置Content-Language
request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性
request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性
update_file_ret = cos_client.update_file(request)
logger.info('update file, return message: ' + str(update_file_ret))
# 5. 更新后再次获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
logger.info('stat file, return message: ' + str(stat_file_ret))
############################################################################
# 目录操作 #
############################################################################
# 1. 生成目录, 目录名为sample_folder
request = CreateFolderRequest(bucket, u'/sample_folder/')
create_folder_ret = cos_client.create_folder(request)
logger.info('create folder ret:' + str(create_folder_ret))
# 2. 更新目录的biz_attr属性
request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录')
update_folder_ret = cos_client.update_folder(request)
logger.info('update folder ret:' + str(update_folder_ret))
# 3. 获取目录属性
request = StatFolderRequest(bucket, u'/sample_folder/')
stat_folder_ret = cos_client.stat_folder(request)
logger.info("stat folder, return message: " + str(stat_folder_ret))
# 4. list目录, 获取目录下的成员
request = ListFolderRequest(bucket, u'/sample_folder/')
list_folder_ret = cos_client.list_folder(request)
logger.info("list folder, return message: " + str(list_folder_ret))
# 5. 删除目录
request = DelFolderRequest(bucket, u'/sample_folder/')
delete_folder_ret = cos_client.del_folder(request)
logger.info("delete folder, return message: " + str(delete_folder_ret))
class CosV4StorageService(storage_service.StorageService):
def __init__(self, *args, **kwargs):
appid = int(kwargs['appid'])
region = kwargs['region']
accesskeyid = unicode(kwargs['accesskeyid'])
accesskeysecret = unicode(kwargs['accesskeysecret'])
bucket = unicode(kwargs['bucket'])
if 'prefix_dir' in kwargs:
self._prefix_dir = kwargs['prefix_dir']
else:
self._prefix_dir = None
# filelist存在时, prefix_dir不生效
if 'filelist' in kwargs:
self._filelist = kwargs['filelist']
self._prefix_dir = None
else:
self._filelist = None
if 'sync_files' in kwargs:
self._sync_files_dir = kwargs['sync_files']
else:
self._sync_files_dir = '/tmp/workspace/sync_files'
self._cos_api = CosClient(appid, accesskeyid, accesskeysecret, region=region)
self._bucket = bucket
self._overwrite = kwargs['overwrite'] == 'true' if 'overwrite' in kwargs else False
self._max_retry = 20
def download(self, task, local_path):
return self._download(task, local_path, False)
def _download(self, task, local_path, is_ignore_size):
for i in range(20):
logger.info("download file with rety {0}".format(i))
import os
try:
os.remove(task.key)
except:
pass
req = DownloadFileRequest(self._bucket, task.key, local_path)
ret = self._cos_api.download_file(req)
logger.debug(str(ret))
if ret['code'] != 0:
logger.error("error code: " + str(ret['code']))
raise IOError("Download Failed, ret=" + str(ret))
if is_ignore_size:
logger.info("Download Successfully, break")
break
if task.size is None:
logger.info("task's size is None, skip check file size on local")
break
from os import path
task_size = 0
if isinstance(task.size, int):
task_size = task.size
else:
task_size = int(task.size['filelen'])
if path.getsize(local_path) != task_size:
logger.error("Download Failed, size1: {size1}, size2: {size2}".format(size1=path.getsize(local_path),
size2=task_size))
else:
logger.info("Download Successfully, break")
break
else:
raise IOError("Download Failed with 20 retry")
def upload(self, task, local_path):
cos_path = task.key
if not cos_path.startswith('/'):
cos_path = '/' + cos_path
if self._prefix_dir:
cos_path = self._prefix_dir + cos_path
if isinstance(local_path, unicode):
local_path.encode('utf-8')
insert_only = 0 if self._overwrite else 1
for i in range(10):
try:
upload_request = UploadFileRequest(self._bucket, unicode(cos_path), local_path, insert_only=insert_only)
upload_file_ret = self._cos_api.upload_file(upload_request)
if upload_file_ret[u'code'] != 0:
logger.warn("upload failed:" + str(upload_file_ret))
raise OSError("UploadError: " + str(upload_file_ret))
else:
break
except:
pass
else:
raise IOError("upload failed")
def list(self, marker):
if self._filelist is not None and len(self._filelist) > 0:
filelist_path = self._sync_files_dir + '/' + os.path.basename(self._filelist)
filelist_task = Task(self._filelist, None, None, None)
self._download(filelist_task, filelist_path, True)
with open(filelist_path) as f:
line = f.readline()
while line:
if not line.startswith('/'):
line = '/' + line
task = self.get_task_by_key(line.strip())
yield task
line = f.readline()
else:
if self._prefix_dir is None:
for i in self.dfs('/'):
yield i
else:
for i in self.dfs(self._prefix_dir):
yield i
def dfs(self, path):
print "DFS: {path}".format(path=to_utf8(path))
_finish = False
_context = u''
max_retry = self._max_retry
path = to_unicode(path)
while not _finish:
request = ListFolderRequest(bucket_name=self._bucket, cos_path=path, context=_context)
request.set_delimiter = '/'
ret = self._cos_api.list_folder(request)
logger.debug(str(ret))
if ret['code'] != 0:
max_retry -= 1
else:
_finish = ret['data']['listover']
_context = ret['data']['context']
for item in ret['data']['infos']:
if 'filelen' in item:
try:
key = "{prefix}{filename}".format(prefix=path, filename=item['name'])
logger.info("key=%s", key)
yield Task(key, item, None, None)
except:
pass
else:
_sub_dir = "{prefix}{filename}".format(prefix=path.encode('utf-8'),
filename=item['name'].encode('utf-8'))
for i in self.dfs(_sub_dir):
yield i
if max_retry == 0:
_finish = True
def exists(self, task):
_path = task.key
_size = task.size
if not _path.startswith('/'):
_path = '/' + _path
logger = getLogger(__name__)
# logger.info("func: exists: " + str(_path))
if self._prefix_dir:
_path = self._prefix_dir + _path
if isinstance(_path, str):
_path = _path.decode('utf-8')
request = StatFileRequest(self._bucket, _path)
ret = self._cos_api.stat_file(request)
logger.debug("ret: " + str(ret))
# import json
# v = json.loads(ret)
if ret['code'] != 0:
# logger.warn("error code: " + str(ret['code']))
return False
if int(ret['data']['filelen']) != int(ret['data']['filesize']):
logger.warn("file is broken, filelen: {len}, filesize: {size}".format(
len=ret['data']['filelen'],
size=ret['data']['filesize']
))
return False
elif _size is not None and int(ret['data']['filelen']) != int(_size):
return False
return True
def get_task_by_key(self, key):
_path = key
if not _path.startswith('/'):
_path = '/' + _path
logger = getLogger(__name__)
if isinstance(_path, str):
_path = _path.decode('utf-8')
request = StatFileRequest(self._bucket, _path)
ret = self._cos_api.stat_file(request)
logger.info("ret: " + str(ret))
# import json
# v = json.loads(ret)
if ret['code'] != 0:
logger.warn("get task by key error, key = {},error code: {}".format(key, str(ret['code'])))
return Task(key, None, None, None)
return Task(key, int(ret['data']['filesize']), None, None)