def dfs(self, path):
print "DFS: {path}".format(path=to_utf8(path))
_finish = False
_context = u''
max_retry = self._max_retry
path = to_unicode(path)
while not _finish:
request = ListFolderRequest(bucket_name=self._bucket, cos_path=path, context=_context)
request.set_delimiter = '/'
ret = self._cos_api.list_folder(request)
logger.debug(str(ret))
if ret['code'] != 0:
max_retry -= 1
else:
_finish = ret['data']['listover']
_context = ret['data']['context']
for item in ret['data']['infos']:
if 'filelen' in item:
try:
key = "{prefix}{filename}".format(prefix=path, filename=item['name'])
logger.info("key=%s", key)
yield Task(key, item, None, None)
except:
pass
else:
_sub_dir = "{prefix}{filename}".format(prefix=path.encode('utf-8'),
filename=item['name'].encode('utf-8'))
for i in self.dfs(_sub_dir):
yield i
if max_retry == 0:
_finish = True
def readCosFiles(cos_client, bucket, subFolder='', debug=1):
start = datetime.now()
if debug: drawTitle('Reading COS files')
cosFilesDict, cosEmptyFolders = {}, []
folderPool = ['/' + subFolder + '/'] if subFolder else ['/']
while len(folderPool) > 0:
folder = folderPool.pop()
cosFilesList, listover, context = [], False, ''
while not listover:
succes = times = 0
while not succes and times < 10:
times += 1
request = ListFolderRequest(bucket, folder)
# 设置请求头(下一页)
if context: request.set_context(context)
try:
list_folder_ret = cos_client.list_folder(
request) #每次请求有199限制
# print(formatJSON(list_folder_ret))
if list_folder_ret['message'] == 'SUCCESS':
succes = 1
cosFilesList += list_folder_ret['data']['infos']
listover = list_folder_ret['data']['listover']
context = list_folder_ret['data']['context']
except:
print('ListFolderRequest Failed.\nFolder: %s\nContext:%s' %
(folder, context))
if times == 10:
print('===Error===: %s info load failed' % (folder))
continue
for item in cosFilesList:
if item['mtime'] == 0: #folder with files
folderPool.append(folder + item['name'])
# print(folder+item['name'])
if ('filesize'
not in item) and (item['mtime'] != 0): #empty folder
cosEmptyFolders.append(folder + item['name'])
if 'filesize' in item: #file
cosFile = re.sub(r'^.*?qcloud.com', '',
item['source_url']) #取含路径的文件名
cosFile = urllib.parse.unquote(cosFile) #url编码中文解码
# print(cosFile,type(cosFile))
cosFilesDict[cosFile] = item['mtime']
if debug:
print('COS files: %s\nCOS empty folders: %s' %
(len(cosFilesDict), len(cosEmptyFolders)))
if cosEmptyFolders:
print('---\ncosEmptyFolders' + formatJSON(cosEmptyFolders))
print('---\n%s: %s' % ('Used', datetime.now() - start))
return cosFilesDict, cosEmptyFolders
def syncLocalToCOS(appid,
secret_id,
secret_key,
bucket,
region_info,
root,
subFolder='',
debug=1):
root = formatPath(root)
subFolder = formatPath(subFolder)
cos_client = CosClient(appid, secret_id, secret_key, region=region_info)
# check cos_client & bucket
list_folder_ret = cos_client.list_folder(ListFolderRequest(bucket, '/'))
if list_folder_ret['message'] != 'SUCCESS':
raise Exception(
'>>> Check your appid / secret_id / secret_key / bucket_name <<<')
# 读取本地需要更新的目录
localFilesDict, localEmptyFolders = readLocalFiles(root,
subFolder,
debug=debug)
# 读取cos上需要更新的目录
cosFilesDict, cosEmptyFolders = readCosFiles(cos_client,
bucket,
subFolder,
debug=debug)
# 比较文件修改时间，筛选出需要更新的文件，并且上传。
modifiedLocalFiles = filterModifiedLocalFiles(localFilesDict,
cosFilesDict,
debug=debug)
if modifiedLocalFiles:
start = datetime.now()
if debug: drawTitle('Uploading files')
for file in modifiedLocalFiles:
uploadToCos(cos_client, bucket, root, file)
if debug: print('---\n%s: %s' % ('Used', datetime.now() - start))
# 筛选出cos上有，但本地已经不存在的文件，删除COS上的文件。
extraCosFiles = filterExtraCosFiles(localFilesDict,
cosFilesDict,
debug=debug)
if extraCosFiles:
start = datetime.now()
if debug: drawTitle('Deleting COS files')
for file in extraCosFiles:
deleteCosFile(cos_client, bucket, file)
if debug: print('---\n%s: %s' % ('Used', datetime.now() - start))
# 同步空文件夹
syncEmptyFolders(cos_client,
bucket,
localEmptyFolders,
cosEmptyFolders,
debug=debug)
def list_folder_ret(path):
request = ListFolderRequest(bucket, u'%s'%path)
list_folder_ret = cos_client.list_folder(request)
# print list_folder_ret['data']['infos'][0]
print list_folder_ret
source_url_list=[]
# print list_folder_ret
# print list_folder_ret['data']['infos']
# print len(list_folder_ret['data']['infos'])
for picture in list_folder_ret['data']['infos']:
print picture
picture_url = picture['name']
source_url_list.append(picture_url)
print len(source_url_list)
return source_url_list
create_folder_ret = self.cos_client.create_folder(request)
result = json.loads(create_folder_ret)
return result
def DeleteFolderTest(self):
request = DelFolderRequest(self.bucket, self.cos_path)
delete_folder_ret = CosClient.del_folder(request)
result = json.loads(delete_folder_ret)
print 'delete folder ret:', repr(delete_folder_ret)
return result
if __name__ == '__main__':
ar = CosClient(appid=AudioRecord.appid,
secret_id=AudioRecord.secret_id,
secret_key=AudioRecord.secret_key)
request = ListFolderRequest(u'porncheck', u'/1400011479/19700101/')
list_folder_ret = ar.list_folder(request)
print list_folder_ret
#request = DelFolderRequest(u'porncheck', u'/1400011479/19700101/')
#delete_folder_ret = ar.del_folder(request)
#print 'delete folder ret:', repr(delete_folder_ret)
#ar = AudioRecord('heydo','/record/introduce/102/')
#ar = AudioRecord('porncheck', '/1400011479/19700101/')
#print ar
#print ar.get_once_sign()
#print ar.DeleteFolderTest()
#print ar.get_once_sign()
#print ar.CreateIntroduceDir()
#cos_demo()
def readCosFiles(cos_client, bucket, subFolder=''):
start = datetime.now()
drawTitle('Reading COS files')
cosFilesDict, cosEmptyFolders = {}, []
if subFolder:
folderPool = [
'/' + re.sub(r'\\', u'/', subFolder).lstrip('\\/').rstrip('\\/') +
'/'
]
else:
folderPool = [u'/']
while len(folderPool) > 0:
folder = folderPool.pop()
cosFilesList, listover, context = [], False, '' #每次请求有199限制 神tm199为毛不是200
while listover == False:
succes = times = 0
while succes == 0 and times < 10:
times += 1
request = ListFolderRequest(bucket, folder)
if context:
request.set_context(
context) #文档里虽然给了参数 但怎么用你猜（文档不会告诉你用set的哦）
try:
list_folder_ret = cos_client.list_folder(request)
# print formatJSON(list_folder_ret)
if list_folder_ret['message'] == u'SUCCESS':
succes = 1
cosFilesList += list_folder_ret['data']['infos']
listover = list_folder_ret['data']['listover']
context = list_folder_ret['data']['context']
except:
print 'ListFolderRequest Failed.\nFolder: %s\nContext:%s' % (
folder, context)
if times == 10:
print '===Error===: %s info load failed' % (folder)
continue
for item in cosFilesList:
if item['mtime'] == 0: #folder with files
folderPool.append(folder + item['name'])
# print folder+item['name']
if ('filesize'
not in item) and (item['mtime'] != 0): #empty folder
cosEmptyFolders.append(folder + item['name'])
if 'filesize' in item: #file
cosFile = re.sub(r'^.*?qcloud.com', '',
item['source_url']) #取含路径的文件名
cosFile = urllib.unquote(cosFile.encode('utf-8')).decode(
'utf-8') #url中文转码
# print cosFile,type(cosFile)
cosFilesDict[cosFile] = item['mtime']
print 'COS files: %s\nCOS empty folders: %s' % (len(cosFilesDict),
len(cosEmptyFolders))
if cosEmptyFolders:
print '---\ncosEmptyFolders' + formatJSON(cosEmptyFolders)
print '---\n%s: %s' % ('Used', datetime.now() - start)
return cosFilesDict, cosEmptyFolders
def cos_demo():
# 设置用户属性, 包括appid, secret_id和secret_key
# 这些属性可以在cos控制台获取(https://console.qcloud.com/cos)
appid = 111 # 替换为用户的appid
secret_id = u'' # 替换为用户的secret_id
secret_key = u'' # 替换为用户的secret_key
region = "shanghai" # # 替换为用户的region，目前可以为 shanghai/guangzhou
cos_client = CosClient(appid, secret_id, secret_key, region)
# 设置要操作的bucket
bucket = u''
############################################################################
# 文件操作 #
############################################################################
# 1. 上传文件(默认不覆盖)
# 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt
# 默认不覆盖, 如果cos上文件存在,则会返回错误
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_1.txt')
upload_file_ret = cos_client.upload_file(request)
logger.info("upload file, return message: " + str(upload_file_ret))
# 分片上传大文件
request = UploadSliceFileRequest(bucket, u'/sample_bigfile.txt',
u'local_bigfile.txt')
ret = cos_client.upload_slice_file(request)
logger.info("slice upload, return message: " + str(ret))
# 2. 上传文件(覆盖文件)
# 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt
with open('local_file_2.txt', 'w') as f:
f.write("hello world2")
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_2.txt')
request.set_insert_only(0) # 设置允许覆盖
upload_file_ret = cos_client.upload_file(request)
logger.info('overwrite file, return message:' + str(upload_file_ret))
# 3. 获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
logger.info('stat file, return message: ' + str(stat_file_ret))
# 4. 更新文件属性
request = UpdateFileRequest(bucket, u'/sample_file.txt')
request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性
request.set_authority(u'eWRPrivate') # 设置文件的权限
request.set_cache_control(u'cache_xxx') # 设置Cache-Control
request.set_content_type(u'application/text') # 设置Content-Type
request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition
request.set_content_language(u'english') # 设置Content-Language
request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性
request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性
update_file_ret = cos_client.update_file(request)
logger.info('update file, return message: ' + str(update_file_ret))
# 5. 更新后再次获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
logger.info('stat file, return message: ' + str(stat_file_ret))
############################################################################
# 目录操作 #
############################################################################
# 1. 生成目录, 目录名为sample_folder
request = CreateFolderRequest(bucket, u'/sample_folder/')
create_folder_ret = cos_client.create_folder(request)
logger.info('create folder ret:' + str(create_folder_ret))
# 2. 更新目录的biz_attr属性
request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录')
update_folder_ret = cos_client.update_folder(request)
logger.info('update folder ret:' + str(update_folder_ret))
# 3. 获取目录属性
request = StatFolderRequest(bucket, u'/sample_folder/')
stat_folder_ret = cos_client.stat_folder(request)
logger.info("stat folder, return message: " + str(stat_folder_ret))
# 4. list目录, 获取目录下的成员
request = ListFolderRequest(bucket, u'/sample_folder/')
list_folder_ret = cos_client.list_folder(request)
logger.info("list folder, return message: " + str(list_folder_ret))
# 5. 删除目录
request = DelFolderRequest(bucket, u'/sample_folder/')
delete_folder_ret = cos_client.del_folder(request)
logger.info("delete folder, return message: " + str(delete_folder_ret))
def cos_demo():
# 设置用户属性, 包括appid, secret_id和secret_key
# 这些属性可以在cos控制台获取(https://console.qcloud.com/cos)
appid = 100000 # 替换为用户的appid
secret_id = u'xxxxxxxx' # 替换为用户的secret_id
secret_key = u'xxxxxxx' # 替换为用户的secret_key
cos_client = CosClient(appid, secret_id, secret_key)
# 设置要操作的bucket
bucket = u'mybucket'
############################################################################
# 文件操作 #
############################################################################
# 1. 上传文件(默认不覆盖)
# 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt
# 默认不覆盖, 如果cos上文件存在,则会返回错误
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_1.txt')
upload_file_ret = cos_client.upload_file(request)
print 'upload file ret:', repr(upload_file_ret)
# 2. 上传文件(覆盖文件)
# 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_2.txt')
request.set_insert_only(0) # 设置允许覆盖
upload_file_ret = cos_client.upload_file(request)
print 'overwrite file ret:', repr(upload_file_ret)
# 3. 获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
print 'stat file ret:', repr(stat_file_ret)
# 4. 更新文件属性
request = UpdateFileRequest(bucket, u'/sample_file.txt')
request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性
request.set_authority(u'eWRPrivate') # 设置文件的权限
request.set_cache_control(u'cache_xxx') # 设置Cache-Control
request.set_content_type(u'application/text') # 设置Content-Type
request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition
request.set_content_language(u'english') # 设置Content-Language
request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性
request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性
update_file_ret = cos_client.update_file(request)
print 'update file ret:', repr(update_file_ret)
# 5. 更新后再次获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
print 'stat file ret:', repr(stat_file_ret)
# 6. 移动文件, 将sample_file.txt移动位sample_file_move.txt
request = MoveFileRequest(bucket, u'/sample_file.txt',
u'/sample_file_move.txt')
stat_file_ret = cos_client.move_file(request)
print 'move file ret:', repr(stat_file_ret)
# 7. 删除文件
request = DelFileRequest(bucket, u'/sample_file_move.txt')
del_ret = cos_client.del_file(request)
print 'del file ret:', repr(del_ret)
############################################################################
# 目录操作 #
############################################################################
# 1. 生成目录, 目录名为sample_folder
request = CreateFolderRequest(bucket, u'/sample_folder/')
create_folder_ret = cos_client.create_folder(request)
print 'create folder ret:', create_folder_ret
# 2. 更新目录的biz_attr属性
request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录')
update_folder_ret = cos_client.update_folder(request)
print 'update folder ret:', repr(update_folder_ret)
# 3. 获取目录属性
request = StatFolderRequest(bucket, u'/sample_folder/')
stat_folder_ret = cos_client.stat_folder(request)
print 'stat folder ret:', repr(stat_folder_ret)
# 4. list目录, 获取目录下的成员
request = ListFolderRequest(bucket, u'/sample_folder/')
list_folder_ret = cos_client.list_folder(request)
print 'list folder ret:', repr(list_folder_ret)
# 5. 删除目录
request = DelFolderRequest(bucket, u'/sample_folder/')
delete_folder_ret = cos_client.del_folder(request)
print 'delete folder ret:', repr(delete_folder_ret)
