def update_packages(url_list):
cos_client = CosClient(app_id, secret_id, secret_key, region)
for package in url_list:
print package.filename
if package.status == 'need update':
print 'update'
elif package.status == 'not exist':
print 'downloading', package.filename
urllib.urlretrieve(package.url,
os.path.join('packages', 'CTS',
package.filename),
reporthook=report)
sys.stdout.flush()
else:
print 'skip', package.filename
request = StatFileRequest(bucket, unicode('/cts/' + package.filename))
stat_file_ret = cos_client.stat_file(request)
if stat_file_ret['code'] != 0:
file_path = os.path.join(os.path.dirname(__file__), 'packages',
'CTS', package.filename)
request = UploadFileRequest(bucket,
unicode('/cts/' + package.filename),
unicode(file_path))
upload_file_ret = cos_client.upload_file(request)
print upload_file_ret
def upload_cos(file):
"""
CVM 上报 https 错误：取消 https 试试
"""
appid = 100000 # 根据自己的情况填写
secret_id = u'# 根据自己的情况填写'
secret_key = u'# 根据自己的情况填写'
region = '# 根据自己的情况填写'
bucket = u'# 根据自己的情况填写'
cos_client = CosClient(appid, secret_id, secret_key, region)
cos_path = u'/' + file
request = UploadFileRequest(bucket, cos_path, unicode(file))
request.set_insert_only(0)
start = time.time()
resp = cos_client.upload_file(request)
# assert resp['code'] == 0
if resp['code'] != 0:
return 0
end = time.time()
elapsed = (end - start) * 1000.0
print u'本次 COS 文件上传时间为 %0.3f ms' % elapsed
return elapsed
def cos_demo(mkdir, file_path):
appid = xxxxxx # 替换为用户的appid
secret_id = u'xxxxxxx' # 替换为用户的secret_id
secret_key = u'xxxxxxx' # 替换为用户的secret_key
region = "tj" # # 替换为用户的region，目前可以为 shanghai/guangzhou
cos_client = CosClient(appid, secret_id, secret_key, region)
bucket = u'xxxx'
request = UploadFileRequest(bucket, u'/' + mkdir + '/' + file_path + '',
u'' + file_path + '')
upload_file_ret = cos_client.upload_file(request)
print logger.info("upload file, return message: " + str(upload_file_ret))
class Client(object):
cos = settings.TECENT_COS
appid, secret_id, secret_key = cos['APP_ID'], cos['SECRET_ID'], cos['SECRET_KEY']
region_info = 'tj'
baseurl = u'http://blog-1251509264.costj.myqcloud.com/'
def __init__(self):
self.bucket = u'blog'
self.client = CosClient(
self.appid,
self.secret_id,
self.secret_key,
region=self.region_info
)
def upload(self,filename):
sep = os.sep
#if sep in filename,meaning that the filename is a absolute file path.
if sep in filename:
filename = os.path.abspath(filename)
#the cos path is the name of file.
cos_path = os.path.basename(filename)
local_path = filename
else:
cos_path = filename
#get the absolute path of the file,if it is only a filename without any path .
local_path = os.path.abspath(filename)
try:
#sometimes,it needed to be encoded to utf-8
cos_path, local_path = unicode('/'+cos_path), unicode(local_path)
except:
cos_path = '/' + cos_path
request = UploadFileRequest(self.bucket,cos_path,local_path)
result = self.client.upload_file(request)
if result[u'code'] == 0:
return result['data']['source_url']
else:
return {'fail':result['message']}
def del_file(self,filename):
if os.path.isdir(filename):
return None
request = DelFileRequest(self.bucket,
unicode('/'+filename))
result = self.client.del_file(request)
print result
return result
def generate_html(packages):
page = markup.page()
page.init(title="CTS Packages", )
page.pre()
for package in packages:
page.a(package.filename, href=cos_info['cos_url'] + package.filename)
page.pre.close()
with open('index.html', 'w') as t:
t.write(str(page))
t.close()
cos_client = CosClient(app_id, secret_id, secret_key, region)
request = UploadFileRequest(bucket, unicode('/cts/index.html'),
unicode('index.html'))
upload_file_ret = cos_client.upload_file(request)
print upload_file_ret
class CosV4StorageService(storage_service.StorageService):
def __init__(self, *args, **kwargs):
appid = int(kwargs['appid'])
region = kwargs['region']
accesskeyid = unicode(kwargs['accesskeyid'])
accesskeysecret = unicode(kwargs['accesskeysecret'])
bucket = unicode(kwargs['bucket'])
if 'prefix_dir' in kwargs:
self._prefix_dir = kwargs['prefix_dir']
else:
self._prefix_dir = None
self._cos_api = CosClient(appid, accesskeyid, accesskeysecret, region=region)
self._bucket = bucket
def download(self, cos_path, local_path):
raise NotImplementedError
def upload(self, cos_path, local_path):
if not cos_path.startswith('/'):
cos_path = '/' + cos_path
if self._prefix_dir:
cos_path = self._prefix_dir + cos_path
if isinstance(local_path, unicode):
local_path.encode('utf-8')
upload_request = UploadFileRequest(self._bucket, unicode(cos_path), local_path)
upload_file_ret = self._cos_api.upload_file(upload_request)
if upload_file_ret[u'code'] != 0:
raise OSError("UploadError: " + str(upload_file_ret))
def list(self):
raise NotImplementedError
def exists(self, _path):
raise NotImplementedError
class UploadUtil(object):
def __init__(self, cos_path, local_path):
appid = 1251316472 # 替换为用户的appid
secret_id = u'AKID3atJdSnNR1Bwf1l4HnnYaSMzpBArC4A5' # 替换为用户的secret_id
secret_key = u'I2J7ho6wpQRRUpdX8K6KISBJCYSlm1hq' # 替换为用户的secret_key
region_info = "sh" # 替换为用户的region，例如 sh 表示华东园区, gz 表示华南园区, tj 表示华北园区
self.cos_client = CosClient(appid,
secret_id,
secret_key,
region=region_info)
self.cos_path = cos_path
self.local_path = local_path
def upload(self, path):
"""
cos_path:/news/jiemian/image/
:param path
:return:
"""
counter = 0
url = ''
while counter != 10:
try:
# 得到hash
uploadName = path.replace('image\\', '')
request = UploadFileRequest(u"crawler",
self.cos_path + uploadName,
self.local_path + path,
insert_only=0)
upload_file_ret = self.cos_client.upload_file(request)
if upload_file_ret['code'] == 0:
data = upload_file_ret['data'] or {}
url = data['source_url']
print u'上传成功 ' + url
else:
print u'上传图片失败', upload_file_ret
break
except Exception as e:
counter += 1
timer_util.sleep(10)
return url
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from qcloud_cos import CosClient
from qcloud_cos import UploadFileRequest
from qcloud_cos import UploadSliceFileRequest
from qcloud_cos import UpdateFileRequest
from qcloud_cos import UpdateFolderRequest
from qcloud_cos import DelFileRequest
from qcloud_cos import DelFolderRequest
from qcloud_cos import CreateFolderRequest
from qcloud_cos import StatFileRequest
from qcloud_cos import StatFolderRequest
from qcloud_cos import ListFolderRequest
import logging
import sys
appid = 1253506722
secret_id = u'AKIDjKjx2vmhePxKldwwKaeVEl5C8vX8pAPS'
secret_key = u'0rVxEKDuOr9JH5h8OnOn026zhT8bswpi'
region_info = "tj"
cos_client = CosClient(appid, secret_id, secret_key, region=region_info)
request = UploadFileRequest(u"codingdie01", sys.argv[2].decode("utf-8"),
sys.argv[1].decode("utf-8"))
result = cos_client.upload_file(request)
print result
class CosV4StorageService(storage_service.StorageService):
def __init__(self, *args, **kwargs):
appid = int(kwargs['appid'])
region = kwargs['region']
accesskeyid = unicode(kwargs['accesskeyid'])
accesskeysecret = unicode(kwargs['accesskeysecret'])
bucket = unicode(kwargs['bucket'])
if 'prefix_dir' in kwargs:
self._prefix_dir = kwargs['prefix_dir']
else:
self._prefix_dir = None
self._cos_api = CosClient(appid,
accesskeyid,
accesskeysecret,
region=region)
self._bucket = bucket
self._overwrite = kwargs[
'overwrite'] == 'true' if 'overwrite' in kwargs else False
self._max_retry = 20
def download(self, task, local_path):
# self._oss_api.get_object_to_file(urllib.unquote(cos_path).encode('utf-8'), local_path)
for i in range(20):
logger.info("download file with rety {0}".format(i))
import os
try:
os.remove(task.key)
except:
pass
req = DownloadFileRequest(self._bucket, task.key, local_path)
ret = self._cos_api.download_file(req)
# self._oss_api.get_object_to_file(task.key, local_path)
logger.debug(str(ret))
if task.size is None:
logger.info(
"task's size is None, skip check file size on local")
break
from os import path
if path.getsize(local_path) != int(task.size):
logger.error(
"Download Failed, size1: {size1}, size2: {size2}".format(
size1=path.getsize(local_path), size2=task.size))
else:
logger.info("Download Successfully, break")
break
else:
raise IOError("Download Failed with 20 retry")
def upload(self, task, local_path):
cos_path = task.key
if not cos_path.startswith('/'):
cos_path = '/' + cos_path
if self._prefix_dir:
cos_path = self._prefix_dir + cos_path
if isinstance(local_path, unicode):
local_path.encode('utf-8')
insert_only = 0 if self._overwrite else 1
for i in range(10):
try:
upload_request = UploadFileRequest(self._bucket,
unicode(cos_path),
local_path,
insert_only=insert_only)
upload_file_ret = self._cos_api.upload_file(upload_request)
if upload_file_ret[u'code'] != 0:
logger.warn("upload failed:" + str(upload_file_ret))
raise OSError("UploadError: " + str(upload_file_ret))
else:
break
except:
pass
else:
raise IOError("upload failed")
def list(self):
if self._prefix_dir is None:
for i in self.dfs('/'):
yield i
else:
for i in self.dfs(self._prefix_dir):
yield i
def dfs(self, path):
print "DFS: {path}".format(path=to_utf8(path))
_finish = False
_context = u''
max_retry = self._max_retry
path = to_unicode(path)
while not _finish:
request = ListFolderRequest(bucket_name=self._bucket,
cos_path=path,
context=_context)
ret = self._cos_api.list_folder(request)
if ret['code'] != 0:
max_retry -= 1
else:
_finish = ret['data']['listover']
_context = ret['data']['context']
for item in ret['data']['infos']:
if 'filelen' in item:
try:
key = "{prefix}{filename}".format(
prefix=path, filename=item['name'])
yield Task(key, item['filelen'], None)
except:
pass
else:
_sub_dir = "{prefix}{filename}".format(
prefix=path.encode('utf-8'),
filename=item['name'].encode('utf-8'))
for i in self.dfs(_sub_dir):
yield i
if max_retry == 0:
_finish = True
def exists(self, task):
_path = task.key
_size = task.size
if not _path.startswith('/'):
_path = '/' + _path
logger = getLogger(__name__)
# logger.info("func: exists: " + str(_path))
if self._prefix_dir:
_path = self._prefix_dir + _path
if isinstance(_path, str):
_path = _path.decode('utf-8')
request = StatFileRequest(self._bucket, _path)
ret = self._cos_api.stat_file(request)
logger.debug("ret: " + str(ret))
# import json
# v = json.loads(ret)
if ret['code'] != 0:
# logger.warn("error code: " + str(ret['code']))
return False
if ret['data']['filelen'] != ret['data']['filesize']:
logger.warn(
"file is broken, filelen: {len}, filesize: {size}".format(
len=ret['data']['filelen'], size=ret['data']['filesize']))
return False
elif _size is not None and ret['data']['filelen'] != _size:
return False
elif ret['data']['filelen'] != _size:
return False
return True
def cos_demo():
# 设置用户属性, 包括appid, secret_id和secret_key
# 这些属性可以在cos控制台获取(https://console.qcloud.com/cos)
appid = 111 # 替换为用户的appid
secret_id = u'' # 替换为用户的secret_id
secret_key = u'' # 替换为用户的secret_key
region = "shanghai" # # 替换为用户的region，目前可以为 shanghai/guangzhou
cos_client = CosClient(appid, secret_id, secret_key, region)
# 设置要操作的bucket
bucket = u''
############################################################################
# 文件操作 #
############################################################################
# 1. 上传文件(默认不覆盖)
# 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt
# 默认不覆盖, 如果cos上文件存在,则会返回错误
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_1.txt')
upload_file_ret = cos_client.upload_file(request)
logger.info("upload file, return message: " + str(upload_file_ret))
# 分片上传大文件
request = UploadSliceFileRequest(bucket, u'/sample_bigfile.txt',
u'local_bigfile.txt')
ret = cos_client.upload_slice_file(request)
logger.info("slice upload, return message: " + str(ret))
# 2. 上传文件(覆盖文件)
# 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt
with open('local_file_2.txt', 'w') as f:
f.write("hello world2")
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_2.txt')
request.set_insert_only(0) # 设置允许覆盖
upload_file_ret = cos_client.upload_file(request)
logger.info('overwrite file, return message:' + str(upload_file_ret))
# 3. 获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
logger.info('stat file, return message: ' + str(stat_file_ret))
# 4. 更新文件属性
request = UpdateFileRequest(bucket, u'/sample_file.txt')
request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性
request.set_authority(u'eWRPrivate') # 设置文件的权限
request.set_cache_control(u'cache_xxx') # 设置Cache-Control
request.set_content_type(u'application/text') # 设置Content-Type
request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition
request.set_content_language(u'english') # 设置Content-Language
request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性
request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性
update_file_ret = cos_client.update_file(request)
logger.info('update file, return message: ' + str(update_file_ret))
# 5. 更新后再次获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
logger.info('stat file, return message: ' + str(stat_file_ret))
############################################################################
# 目录操作 #
############################################################################
# 1. 生成目录, 目录名为sample_folder
request = CreateFolderRequest(bucket, u'/sample_folder/')
create_folder_ret = cos_client.create_folder(request)
logger.info('create folder ret:' + str(create_folder_ret))
# 2. 更新目录的biz_attr属性
request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录')
update_folder_ret = cos_client.update_folder(request)
logger.info('update folder ret:' + str(update_folder_ret))
# 3. 获取目录属性
request = StatFolderRequest(bucket, u'/sample_folder/')
stat_folder_ret = cos_client.stat_folder(request)
logger.info("stat folder, return message: " + str(stat_folder_ret))
# 4. list目录, 获取目录下的成员
request = ListFolderRequest(bucket, u'/sample_folder/')
list_folder_ret = cos_client.list_folder(request)
logger.info("list folder, return message: " + str(list_folder_ret))
# 5. 删除目录
request = DelFolderRequest(bucket, u'/sample_folder/')
delete_folder_ret = cos_client.del_folder(request)
logger.info("delete folder, return message: " + str(delete_folder_ret))
def cos_demo():
# 设置用户属性, 包括appid, secret_id和secret_key
# 这些属性可以在cos控制台获取(https://console.qcloud.com/cos)
appid = 100000 # 替换为用户的appid
secret_id = u'xxxxxxxx' # 替换为用户的secret_id
secret_key = u'xxxxxxx' # 替换为用户的secret_key
cos_client = CosClient(appid, secret_id, secret_key)
# 设置要操作的bucket
bucket = u'mybucket'
############################################################################
# 文件操作 #
############################################################################
# 1. 上传文件(默认不覆盖)
# 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt
# 默认不覆盖, 如果cos上文件存在,则会返回错误
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_1.txt')
upload_file_ret = cos_client.upload_file(request)
print 'upload file ret:', repr(upload_file_ret)
# 2. 上传文件(覆盖文件)
# 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_2.txt')
request.set_insert_only(0) # 设置允许覆盖
upload_file_ret = cos_client.upload_file(request)
print 'overwrite file ret:', repr(upload_file_ret)
# 3. 获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
print 'stat file ret:', repr(stat_file_ret)
# 4. 更新文件属性
request = UpdateFileRequest(bucket, u'/sample_file.txt')
request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性
request.set_authority(u'eWRPrivate') # 设置文件的权限
request.set_cache_control(u'cache_xxx') # 设置Cache-Control
request.set_content_type(u'application/text') # 设置Content-Type
request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition
request.set_content_language(u'english') # 设置Content-Language
request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性
request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性
update_file_ret = cos_client.update_file(request)
print 'update file ret:', repr(update_file_ret)
# 5. 更新后再次获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
print 'stat file ret:', repr(stat_file_ret)
# 6. 移动文件, 将sample_file.txt移动位sample_file_move.txt
request = MoveFileRequest(bucket, u'/sample_file.txt',
u'/sample_file_move.txt')
stat_file_ret = cos_client.move_file(request)
print 'move file ret:', repr(stat_file_ret)
# 7. 删除文件
request = DelFileRequest(bucket, u'/sample_file_move.txt')
del_ret = cos_client.del_file(request)
print 'del file ret:', repr(del_ret)
############################################################################
# 目录操作 #
############################################################################
# 1. 生成目录, 目录名为sample_folder
request = CreateFolderRequest(bucket, u'/sample_folder/')
create_folder_ret = cos_client.create_folder(request)
print 'create folder ret:', create_folder_ret
# 2. 更新目录的biz_attr属性
request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录')
update_folder_ret = cos_client.update_folder(request)
print 'update folder ret:', repr(update_folder_ret)
# 3. 获取目录属性
request = StatFolderRequest(bucket, u'/sample_folder/')
stat_folder_ret = cos_client.stat_folder(request)
print 'stat folder ret:', repr(stat_folder_ret)
# 4. list目录, 获取目录下的成员
request = ListFolderRequest(bucket, u'/sample_folder/')
list_folder_ret = cos_client.list_folder(request)
print 'list folder ret:', repr(list_folder_ret)
# 5. 删除目录
request = DelFolderRequest(bucket, u'/sample_folder/')
delete_folder_ret = cos_client.del_folder(request)
print 'delete folder ret:', repr(delete_folder_ret)
#!/usr/bin/env python
# coding=utf-8
from qcloud_cos import CosClient
from qcloud_cos import UploadFileRequest
appid = 100000 # 替换为你的appid
secret_id = u'' # 替换为你的secret_id
secret_key = u'' # 替换为你的secret_key
bucket = u'' # 替换为你要操作的bucket名
region = "tj" # 替换为该bucket所属的地区代码, tj/sh/gz/sgp
cos_client = CosClient(appid, secret_id, secret_key, region)
##################################
# 文件操作 #
##################################
# 上传文件(默认不允许覆盖)
# 将本地的 upload_example.txt 上传到 bucket 的根分区下,并命名为 upload_sample_python.txt
request = UploadFileRequest(bucket, u'/upload_sample_python.txt',
u'/data/upload/upload_example.txt')
upload_file_response = cos_client.upload_file(request)
print upload_file_response
class UploadUtil(object):
def __init__(self, cos_path, local_path):
appid = 1251316472 # 替换为用户的appid
secret_id = u'AKID3atJdSnNR1Bwf1l4HnnYaSMzpBArC4A5' # 替换为用户的secret_id
secret_key = u'I2J7ho6wpQRRUpdX8K6KISBJCYSlm1hq' # 替换为用户的secret_key
region_info = "sh" # 替换为用户的region，例如 sh 表示华东园区, gz 表示华南园区, tj 表示华北园区
self.cos_client = CosClient(appid,
secret_id,
secret_key,
region=region_info)
self.cos_path = cos_path
self.local_path = local_path
def uploadList(self, listFile):
"""
cos_path:/news/jiemian/image/
:param listFile: [{url,path},{}]
:return:
"""
results = []
for item in listFile:
# 得到hash
path = item['path']
uploadName = path.replace(u'full/', '')
request = UploadFileRequest(u"crawler",
self.cos_path + uploadName,
self.local_path + path,
insert_only=0)
upload_file_ret = self.cos_client.upload_file(request)
if upload_file_ret['code'] == 0:
data = upload_file_ret['data'] or {}
url = data['source_url']
results.append(url)
print u'upload successfully', url
else:
print u'fail to uplaod ', upload_file_ret
return results
def upload(self, path):
"""
cos_path:/news/jiemian/image/
:param path
:return:
"""
counter = 0
url = ''
while counter != 10:
try:
# 得到hash
uploadName = path.replace('full/', '')
request = UploadFileRequest(u"crawler",
self.cos_path + uploadName,
self.local_path + path,
insert_only=0)
upload_file_ret = self.cos_client.upload_file(request)
if upload_file_ret['code'] == 0:
data = upload_file_ret['data'] or {}
url = data['source_url']
print u'上传成功', url
else:
print u'上传图片失败', upload_file_ret
break
except Exception as e:
counter += 1
TimerUtil.sleep(10)
return url
def cos_demo():
# 设置用户属性, 包括appid, secret_id和secret_key
# 这些属性可以在cos控制台获取(https://console.qcloud.com/cos)
appid = 111 # 替换为用户的appid
secret_id = u'' # 替换为用户的secret_id
secret_key = u'' # 替换为用户的secret_key
region = "shanghai" # # 替换为用户的region，目前可以为 shanghai/guangzhou
cos_client = CosClient(appid, secret_id, secret_key, region)
# 设置要操作的bucket
bucket = u''
############################################################################
# 文件操作 #
############################################################################
# 1. 上传文件(默认不覆盖)
# 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt
# 默认不覆盖, 如果cos上文件存在,则会返回错误
request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_1.txt')
upload_file_ret = cos_client.upload_file(request)
logger.info("upload file, return message: " + str(upload_file_ret))
# 分片上传大文件
request = UploadSliceFileRequest(bucket, u'/sample_bigfile.txt', u'local_bigfile.txt')
ret = cos_client.upload_slice_file(request)
logger.info("slice upload, return message: " + str(ret))
# 2. 上传文件(覆盖文件)
# 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt
with open('local_file_2.txt', 'w') as f:
f.write("hello world2")
request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_2.txt')
request.set_insert_only(0) # 设置允许覆盖
upload_file_ret = cos_client.upload_file(request)
logger.info('overwrite file, return message:' + str(upload_file_ret))
# 3. 获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
logger.info('stat file, return message: ' + str(stat_file_ret))
# 4. 更新文件属性
request = UpdateFileRequest(bucket, u'/sample_file.txt')
request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性
request.set_authority(u'eWRPrivate') # 设置文件的权限
request.set_cache_control(u'cache_xxx') # 设置Cache-Control
request.set_content_type(u'application/text') # 设置Content-Type
request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition
request.set_content_language(u'english') # 设置Content-Language
request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性
request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性
update_file_ret = cos_client.update_file(request)
logger.info('update file, return message: ' + str(update_file_ret))
# 5. 更新后再次获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
logger.info('stat file, return message: ' + str(stat_file_ret))
############################################################################
# 目录操作 #
############################################################################
# 1. 生成目录, 目录名为sample_folder
request = CreateFolderRequest(bucket, u'/sample_folder/')
create_folder_ret = cos_client.create_folder(request)
logger.info('create folder ret:' + str(create_folder_ret))
# 2. 更新目录的biz_attr属性
request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录')
update_folder_ret = cos_client.update_folder(request)
logger.info('update folder ret:' + str(update_folder_ret))
# 3. 获取目录属性
request = StatFolderRequest(bucket, u'/sample_folder/')
stat_folder_ret = cos_client.stat_folder(request)
logger.info("stat folder, return message: " + str(stat_folder_ret))
# 4. list目录, 获取目录下的成员
request = ListFolderRequest(bucket, u'/sample_folder/')
list_folder_ret = cos_client.list_folder(request)
logger.info("list folder, return message: " + str(list_folder_ret))
# 5. 删除目录
request = DelFolderRequest(bucket, u'/sample_folder/')
delete_folder_ret = cos_client.del_folder(request)
logger.info("delete folder, return message: " + str(delete_folder_ret))
from qcloud_cos import UploadFileRequest
from qcloud_cos import CosClient
import sys
if __name__ == '__main__':
if len(sys.argv) < 3:
print "argv error"
bucket = u'koprvhdix117'
app_id = 10038234
secret_id = u'AKIDkTQGMuCeJvtTTlqg911BfF393ghumqHp'
secret_key = u'ZE1uBa6jfbsB0vVyfbWhw5JuZKPwaEwh'
cos_client = CosClient(app_id, secret_id, secret_key)
if sys.argv[1] == "upload":
cos_path = unicode(sys.argv[2])
local_path = unicode(sys.argv[3])
request = UploadFileRequest(bucket, cos_path, local_path)
upload_file_ret = cos_client.upload_file(request)
print upload_file_ret
elif sys.argv[1] == "delete":
cos_path = unicode(sys.argv[2])
request = DelFileRequest(bucket, cos_path)
delete_file_ret = cos_client.del_file(request)
print delete_file_ret
else:
print "argv error"
def cos_demo():
# 设置用户属性, 包括appid, secret_id和secret_key
# 这些属性可以在cos控制台获取(https://console.qcloud.com/cos)
appid = 100000 # 替换为用户的appid
secret_id = u"xxxxxxxx" # 替换为用户的secret_id
secret_key = u"xxxxxxx" # 替换为用户的secret_key
cos_client = CosClient(appid, secret_id, secret_key)
# 设置要操作的bucket
bucket = u"mybucket"
############################################################################
# 文件操作 #
############################################################################
# 1. 上传文件(默认不覆盖)
# 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt
# 默认不覆盖, 如果cos上文件存在,则会返回错误
request = UploadFileRequest(bucket, u"/sample_file.txt", u"local_file_1.txt")
upload_file_ret = cos_client.upload_file(request)
print "upload file ret:", repr(upload_file_ret)
# 2. 上传文件(覆盖文件)
# 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt
request = UploadFileRequest(bucket, u"/sample_file.txt", u"local_file_2.txt")
request.set_insert_only(0) # 设置允许覆盖
upload_file_ret = cos_client.upload_file(request)
print "overwrite file ret:", repr(upload_file_ret)
# 3. 获取文件属性
request = StatFileRequest(bucket, u"/sample_file.txt")
stat_file_ret = cos_client.stat_file(request)
print "stat file ret:", repr(stat_file_ret)
# 4. 更新文件属性
request = UpdateFileRequest(bucket, u"/sample_file.txt")
request.set_biz_attr(u"这是个demo文件") # 设置文件biz_attr属性
request.set_authority(u"eWRPrivate") # 设置文件的权限
request.set_cache_control(u"cache_xxx") # 设置Cache-Control
request.set_content_type(u"application/text") # 设置Content-Type
request.set_content_disposition(u"ccccxxx.txt") # 设置Content-Disposition
request.set_content_language(u"english") # 设置Content-Language
request.set_x_cos_meta(u"x-cos-meta-xxx", u"xxx") # 设置自定义的x-cos-meta-属性
request.set_x_cos_meta(u"x-cos-meta-yyy", u"yyy") # 设置自定义的x-cos-meta-属性
update_file_ret = cos_client.update_file(request)
print "update file ret:", repr(update_file_ret)
# 5. 更新后再次获取文件属性
request = StatFileRequest(bucket, u"/sample_file.txt")
stat_file_ret = cos_client.stat_file(request)
print "stat file ret:", repr(stat_file_ret)
# 6. 移动文件, 将sample_file.txt移动位sample_file_move.txt
request = MoveFileRequest(bucket, u"/sample_file.txt", u"/sample_file_move.txt")
stat_file_ret = cos_client.move_file(request)
print "move file ret:", repr(stat_file_ret)
# 7. 删除文件
request = DelFileRequest(bucket, u"/sample_file_move.txt")
del_ret = cos_client.del_file(request)
print "del file ret:", repr(del_ret)
############################################################################
# 目录操作 #
############################################################################
# 1. 生成目录, 目录名为sample_folder
request = CreateFolderRequest(bucket, u"/sample_folder/")
create_folder_ret = cos_client.create_folder(request)
print "create folder ret:", create_folder_ret
# 2. 更新目录的biz_attr属性
request = UpdateFolderRequest(bucket, u"/sample_folder/", u"这是一个测试目录")
update_folder_ret = cos_client.update_folder(request)
print "update folder ret:", repr(update_folder_ret)
# 3. 获取目录属性
request = StatFolderRequest(bucket, u"/sample_folder/")
stat_folder_ret = cos_client.stat_folder(request)
print "stat folder ret:", repr(stat_folder_ret)
# 4. list目录, 获取目录下的成员
request = ListFolderRequest(bucket, u"/sample_folder/")
list_folder_ret = cos_client.list_folder(request)
print "list folder ret:", repr(list_folder_ret)
# 5. 删除目录
request = DelFolderRequest(bucket, u"/sample_folder/")
delete_folder_ret = cos_client.del_folder(request)
print "delete folder ret:", repr(delete_folder_ret)
class CosFS(object):
def __init__(self, appid, secret_id, secret_key, bucket, region = None):
self.appid = appid
self.secret_id = secret_id
self.secret_key = secret_key
self.bucket = bucket
if region:
self.cos_client = CosClient(appid, secret_id, secret_key, region=region)
else:
self.cos_client = CosClient(appid, secret_id, secret_key)
def list_dir(self, path=u'/'):
path = to_unicode(path)
prefix = u''
if path.endswith(u'*'):
prefix = os.path.basename(path)[:-1]
path = os.path.dirname(path)
if not path.endswith(u'/'):
path += u'/'
context=u''
arr_data = []
while True:
request = ListFolderRequest(self.bucket, path, prefix=prefix, context=context)
result = self.cos_client.list_folder(request)
if result['code'] != 0:
raise CosFSException(result['code'], result['message'] + ': ' + path)
data = result['data']
arr_data.append(data)
has_more = False
if 'has_more' in data:
has_more = data['has_more']
if 'listover' in data: #cos v4
has_more = not data['listover']
if has_more:
context = data['context']
else:
break
dataset = {'dircount': 0, 'filecount': 0, 'infos': []}
for data in arr_data:
if 'filecount' not in data:
data['filecount'] = 0
data['dircount'] = 0
for entry in data['infos']:
if self.isFile(entry):
data['filecount'] += 1
else:
entry['name'] = entry['name'].rstrip(u'/')
data['dircount'] += 1
dataset['dircount'] += data['dircount']
dataset['filecount'] += data['filecount']
dataset['infos'] += data['infos']
return dataset
def ls(self, path=u'/', detail=False, recursive=False):
content = self.list_dir(path)
if recursive:
print '%s:' % (path.encode('utf-8'))
for entry in content['infos']:
isFile = self.isFile(entry)
name = entry['name'].encode('utf-8')
if detail:
ctime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(entry['ctime'])))
print './%s%s: [size:%d] [created_at:%s]' % (name, '' if isFile else '/', entry['filesize'] if isFile else 0, ctime)
else:
print name + ('' if isFile else '/')
if recursive and content['dircount'] > 0:
print ''
for info in content['infos']:
if 'sha' not in info:
self.ls(path.rstrip(u'/') + u'/' + info['name'], detail, recursive)
print ''
def mv(self, src, dest, overwrite=False):
src = to_unicode(src)
dest = to_unicode(dest)
if dest.endswith(u'/'):
dest += os.path.basename(src)
request = MoveFileRequest(self.bucket, src, dest)
result = self.cos_client.move_file(request)
if result['code'] != 0:
raise CosFSException(result['code'], result['message'])
def download(self, remote, local, overwrite=False):
local = to_unicode(local)
remote = to_unicode(remote)
if local == u'.':
local += u'/'
if local.endswith(u'/'):
local += os.path.basename(remote)
if os.path.exists(local):
if overwrite:
os.unlink(local)
else:
raise CosFSException(-1, 'local file %s exists' % local)
fileattr = self.stat(remote)
url = fileattr['source_url'] + '?sign=' + fileattr['sign']
download_file(url, local)
def cat(self, path):
fileattr = self.stat(to_unicode(path))
url = fileattr['source_url'] + '?sign=' + fileattr['sign']
download_file(url, '/dev/stdout')
def upload(self, local, remote, overwrite=False, silent=False):
local = to_unicode(local)
remote = to_unicode(remote)
if remote.endswith(u'/'):
remote += os.path.basename(local)
request = UploadFileRequest(self.bucket, remote, local)
if overwrite:
request.set_insert_only(0)
result = self.cos_client.upload_file(request)
if result['code'] != 0:
if result['code'] == CODE_SAME_FILE:
if not silent:
print >>sys.stderr, "skipped: same file on COS"
return
if overwrite and result['code'] == CODE_ERR_OFF_GOBACK:
print >>sys.stderr, "fix tencent bug: remove remote file when ErrOffGoBack occurs"
"""
上传出现这个4024的错误之后，覆盖（insertonly=0）参数也不能成功，只能删除文件后重新上传
建议您将分片大小改为1M 分片上传之间sleep 100ms 出现错误的概率会小很多
抱歉，这个问题暂时无法彻底解决，给您带来了不便。
"""
self.rm(remote)
raise CosFSException(result['code'], result['message'])
def cp(self, src, dest, overwrite=False):
if src.startswith('cos:') and dest.startswith('cos:'): #move in cos
raise CosFSException(-1, "not supported yet, use mv please")
elif src.startswith('cos:'): #download
self.download(src[4:], dest, overwrite)
elif dest.startswith('cos:'): #upload
self.upload(src, dest[4:], overwrite)
else:
raise CosFSException(-1, "at least one of src/dest should start with `cos:`")
def cpdir(self, src, dest, conflict=CONFLICT_ERROR):
begin_at = time.time()
src = to_unicode(src)
dest = to_unicode(dest)
if src.startswith(u'cos:') and dest.startswith(u'cos:'):
raise CosFSException(-1, "not supported")
elif src.startswith(u'cos:'): #download
self.downloadDir(src[4:], dest, conflict)
elif dest.startswith(u'cos:'): #upload
self.uploadDir(src, dest[4:], conflict)
else:
raise CosFSException(-1, "at least one of src/dest should start with `cos:`")
usage = time.time() - begin_at
#print '[Time Usage: %.6f]' % usage
#将[cos上remote目录里]的内容下载到[本地local目录里]，如果local目录不存在，会被创建
def downloadDir(self, remote, local, conflict):
remote = remote.rstrip(u'/')
local = local.rstrip(u'/')
file_queue = Queue.Queue()
def walk(path = u'/', level = 0):
print '[mkdir] ' + ' ' * level + local + path
localMkdir(local + path)
content = self.list_dir(remote + path)
for entry in content['infos']:
name = path + entry['name'].encode('utf-8')
if self.isFile(entry):
print '[copy] ' + ' ' * level + local + name
overwrite = conflict == CONFLICT_OVERWRITE
file_queue.put([self.download, (remote + name, local + name, overwrite)])
else:
walk(name + '/', level + 1)
walk()
CosThread.execute(file_queue)
print >>sys.stderr, "[download finished]"
def uploadDir(self, local, remote, conflict):
if not remote.endswith(u'/'):
remote += u'/'
if not local.endswith(u'/'):
remote += os.path.basename(local) + u'/'
local = os.path.abspath(local) + u'/'
if not os.path.isdir(local):
raise CosFSException(-1, "please specify a local directory")
retry(self.mkdir, remote)
def uploadFile(localfile, remotefile):
try:
print >>sys.stderr, '[uploadFile] %s => %s' % (localfile, remotefile)
self.upload(localfile, remotefile, silent=True)
print >>sys.stderr, 'done: new'
except Exception, e:
if e[0] == CODE_EXISTED: #ERROR_CMD_COS_FILE_EXIST
if conflict == CONFLICT_SKIP:
print >>sys.stderr, 'skipped: existed'
elif conflict == CONFLICT_OVERWRITE:
self.upload(localfile, remotefile, overwrite=True)
print >>sys.stderr, 'done: overwrite'
else:
raise
return
raise
file_queue = Queue.Queue()
def process_dir(arg, dirname, filelist):
dir_suffix = dirname[len(local):]
remote_dir = remote + dir_suffix
file_queue.put([self.mkdir, (remote_dir,)])
print >>sys.stderr, '[doUpload] queue for mkdir %s' % remote_dir
for filename in filelist:
localfile = dirname.rstrip(u'/') + u'/' + filename
if os.path.isdir(localfile):
continue
if os.path.islink(localfile):
print >>sys.stderr, '[doUpload] skip symlink %s' % (localfile)
continue
remotefile = remote_dir.rstrip(u'/') + u'/' + filename
file_queue.put([uploadFile, (localfile, remotefile)])
print >>sys.stderr, '[doUpload] queue for upload file %s ...' % remotefile
os.path.walk(local, process_dir, None)
CosThread.execute(file_queue)
print >>sys.stderr, "[upload finished]"
class CosV4StorageService(storage_service.StorageService):
def __init__(self, *args, **kwargs):
appid = int(kwargs['appid'])
region = kwargs['region']
accesskeyid = unicode(kwargs['accesskeyid'])
accesskeysecret = unicode(kwargs['accesskeysecret'])
bucket = unicode(kwargs['bucket'])
if 'prefix_dir' in kwargs:
self._prefix_dir = kwargs['prefix_dir']
else:
self._prefix_dir = None
# filelist存在时, prefix_dir不生效
if 'filelist' in kwargs:
self._filelist = kwargs['filelist']
self._prefix_dir = None
else:
self._filelist = None
if 'sync_files' in kwargs:
self._sync_files_dir = kwargs['sync_files']
else:
self._sync_files_dir = '/tmp/workspace/sync_files'
self._cos_api = CosClient(appid, accesskeyid, accesskeysecret, region=region)
self._bucket = bucket
self._overwrite = kwargs['overwrite'] == 'true' if 'overwrite' in kwargs else False
self._max_retry = 20
def download(self, task, local_path):
return self._download(task, local_path, False)
def _download(self, task, local_path, is_ignore_size):
for i in range(20):
logger.info("download file with rety {0}".format(i))
import os
try:
os.remove(task.key)
except:
pass
req = DownloadFileRequest(self._bucket, task.key, local_path)
ret = self._cos_api.download_file(req)
logger.debug(str(ret))
if ret['code'] != 0:
logger.error("error code: " + str(ret['code']))
raise IOError("Download Failed, ret=" + str(ret))
if is_ignore_size:
logger.info("Download Successfully, break")
break
if task.size is None:
logger.info("task's size is None, skip check file size on local")
break
from os import path
task_size = 0
if isinstance(task.size, int):
task_size = task.size
else:
task_size = int(task.size['filelen'])
if path.getsize(local_path) != task_size:
logger.error("Download Failed, size1: {size1}, size2: {size2}".format(size1=path.getsize(local_path),
size2=task_size))
else:
logger.info("Download Successfully, break")
break
else:
raise IOError("Download Failed with 20 retry")
def upload(self, task, local_path):
cos_path = task.key
if not cos_path.startswith('/'):
cos_path = '/' + cos_path
if self._prefix_dir:
cos_path = self._prefix_dir + cos_path
if isinstance(local_path, unicode):
local_path.encode('utf-8')
insert_only = 0 if self._overwrite else 1
for i in range(10):
try:
upload_request = UploadFileRequest(self._bucket, unicode(cos_path), local_path, insert_only=insert_only)
upload_file_ret = self._cos_api.upload_file(upload_request)
if upload_file_ret[u'code'] != 0:
logger.warn("upload failed:" + str(upload_file_ret))
raise OSError("UploadError: " + str(upload_file_ret))
else:
break
except:
pass
else:
raise IOError("upload failed")
def list(self, marker):
if self._filelist is not None and len(self._filelist) > 0:
filelist_path = self._sync_files_dir + '/' + os.path.basename(self._filelist)
filelist_task = Task(self._filelist, None, None, None)
self._download(filelist_task, filelist_path, True)
with open(filelist_path) as f:
line = f.readline()
while line:
if not line.startswith('/'):
line = '/' + line
task = self.get_task_by_key(line.strip())
yield task
line = f.readline()
else:
if self._prefix_dir is None:
for i in self.dfs('/'):
yield i
else:
for i in self.dfs(self._prefix_dir):
yield i
def dfs(self, path):
print "DFS: {path}".format(path=to_utf8(path))
_finish = False
_context = u''
max_retry = self._max_retry
path = to_unicode(path)
while not _finish:
request = ListFolderRequest(bucket_name=self._bucket, cos_path=path, context=_context)
request.set_delimiter = '/'
ret = self._cos_api.list_folder(request)
logger.debug(str(ret))
if ret['code'] != 0:
max_retry -= 1
else:
_finish = ret['data']['listover']
_context = ret['data']['context']
for item in ret['data']['infos']:
if 'filelen' in item:
try:
key = "{prefix}{filename}".format(prefix=path, filename=item['name'])
logger.info("key=%s", key)
yield Task(key, item, None, None)
except:
pass
else:
_sub_dir = "{prefix}{filename}".format(prefix=path.encode('utf-8'),
filename=item['name'].encode('utf-8'))
for i in self.dfs(_sub_dir):
yield i
if max_retry == 0:
_finish = True
def exists(self, task):
_path = task.key
_size = task.size
if not _path.startswith('/'):
_path = '/' + _path
logger = getLogger(__name__)
# logger.info("func: exists: " + str(_path))
if self._prefix_dir:
_path = self._prefix_dir + _path
if isinstance(_path, str):
_path = _path.decode('utf-8')
request = StatFileRequest(self._bucket, _path)
ret = self._cos_api.stat_file(request)
logger.debug("ret: " + str(ret))
# import json
# v = json.loads(ret)
if ret['code'] != 0:
# logger.warn("error code: " + str(ret['code']))
return False
if int(ret['data']['filelen']) != int(ret['data']['filesize']):
logger.warn("file is broken, filelen: {len}, filesize: {size}".format(
len=ret['data']['filelen'],
size=ret['data']['filesize']
))
return False
elif _size is not None and int(ret['data']['filelen']) != int(_size):
return False
return True
def get_task_by_key(self, key):
_path = key
if not _path.startswith('/'):
_path = '/' + _path
logger = getLogger(__name__)
if isinstance(_path, str):
_path = _path.decode('utf-8')
request = StatFileRequest(self._bucket, _path)
ret = self._cos_api.stat_file(request)
logger.info("ret: " + str(ret))
# import json
# v = json.loads(ret)
if ret['code'] != 0:
logger.warn("get task by key error, key = {},error code: {}".format(key, str(ret['code'])))
return Task(key, None, None, None)
return Task(key, int(ret['data']['filesize']), None, None)
