def update_packages(url_list): cos_client = CosClient(app_id, secret_id, secret_key, region) for package in url_list: print package.filename if package.status == 'need update': print 'update' elif package.status == 'not exist': print 'downloading', package.filename urllib.urlretrieve(package.url, os.path.join('packages', 'CTS', package.filename), reporthook=report) sys.stdout.flush() else: print 'skip', package.filename request = StatFileRequest(bucket, unicode('/cts/' + package.filename)) stat_file_ret = cos_client.stat_file(request) if stat_file_ret['code'] != 0: file_path = os.path.join(os.path.dirname(__file__), 'packages', 'CTS', package.filename) request = UploadFileRequest(bucket, unicode('/cts/' + package.filename), unicode(file_path)) upload_file_ret = cos_client.upload_file(request) print upload_file_ret
def upload_cos(file): """ CVM 上报 https 错误:取消 https 试试 """ appid = 100000 # 根据自己的情况填写 secret_id = u'# 根据自己的情况填写' secret_key = u'# 根据自己的情况填写' region = '# 根据自己的情况填写' bucket = u'# 根据自己的情况填写' cos_client = CosClient(appid, secret_id, secret_key, region) cos_path = u'/' + file request = UploadFileRequest(bucket, cos_path, unicode(file)) request.set_insert_only(0) start = time.time() resp = cos_client.upload_file(request) # assert resp['code'] == 0 if resp['code'] != 0: return 0 end = time.time() elapsed = (end - start) * 1000.0 print u'本次 COS 文件上传时间为 %0.3f ms' % elapsed return elapsed
def cos_demo(mkdir, file_path): appid = xxxxxx # 替换为用户的appid secret_id = u'xxxxxxx' # 替换为用户的secret_id secret_key = u'xxxxxxx' # 替换为用户的secret_key region = "tj" # # 替换为用户的region,目前可以为 shanghai/guangzhou cos_client = CosClient(appid, secret_id, secret_key, region) bucket = u'xxxx' request = UploadFileRequest(bucket, u'/' + mkdir + '/' + file_path + '', u'' + file_path + '') upload_file_ret = cos_client.upload_file(request) print logger.info("upload file, return message: " + str(upload_file_ret))
class Client(object): cos = settings.TECENT_COS appid, secret_id, secret_key = cos['APP_ID'], cos['SECRET_ID'], cos['SECRET_KEY'] region_info = 'tj' baseurl = u'http://blog-1251509264.costj.myqcloud.com/' def __init__(self): self.bucket = u'blog' self.client = CosClient( self.appid, self.secret_id, self.secret_key, region=self.region_info ) def upload(self,filename): sep = os.sep #if sep in filename,meaning that the filename is a absolute file path. if sep in filename: filename = os.path.abspath(filename) #the cos path is the name of file. cos_path = os.path.basename(filename) local_path = filename else: cos_path = filename #get the absolute path of the file,if it is only a filename without any path . local_path = os.path.abspath(filename) try: #sometimes,it needed to be encoded to utf-8 cos_path, local_path = unicode('/'+cos_path), unicode(local_path) except: cos_path = '/' + cos_path request = UploadFileRequest(self.bucket,cos_path,local_path) result = self.client.upload_file(request) if result[u'code'] == 0: return result['data']['source_url'] else: return {'fail':result['message']} def del_file(self,filename): if os.path.isdir(filename): return None request = DelFileRequest(self.bucket, unicode('/'+filename)) result = self.client.del_file(request) print result return result
def generate_html(packages): page = markup.page() page.init(title="CTS Packages", ) page.pre() for package in packages: page.a(package.filename, href=cos_info['cos_url'] + package.filename) page.pre.close() with open('index.html', 'w') as t: t.write(str(page)) t.close() cos_client = CosClient(app_id, secret_id, secret_key, region) request = UploadFileRequest(bucket, unicode('/cts/index.html'), unicode('index.html')) upload_file_ret = cos_client.upload_file(request) print upload_file_ret
class CosV4StorageService(storage_service.StorageService): def __init__(self, *args, **kwargs): appid = int(kwargs['appid']) region = kwargs['region'] accesskeyid = unicode(kwargs['accesskeyid']) accesskeysecret = unicode(kwargs['accesskeysecret']) bucket = unicode(kwargs['bucket']) if 'prefix_dir' in kwargs: self._prefix_dir = kwargs['prefix_dir'] else: self._prefix_dir = None self._cos_api = CosClient(appid, accesskeyid, accesskeysecret, region=region) self._bucket = bucket def download(self, cos_path, local_path): raise NotImplementedError def upload(self, cos_path, local_path): if not cos_path.startswith('/'): cos_path = '/' + cos_path if self._prefix_dir: cos_path = self._prefix_dir + cos_path if isinstance(local_path, unicode): local_path.encode('utf-8') upload_request = UploadFileRequest(self._bucket, unicode(cos_path), local_path) upload_file_ret = self._cos_api.upload_file(upload_request) if upload_file_ret[u'code'] != 0: raise OSError("UploadError: " + str(upload_file_ret)) def list(self): raise NotImplementedError def exists(self, _path): raise NotImplementedError
class UploadUtil(object): def __init__(self, cos_path, local_path): appid = 1251316472 # 替换为用户的appid secret_id = u'AKID3atJdSnNR1Bwf1l4HnnYaSMzpBArC4A5' # 替换为用户的secret_id secret_key = u'I2J7ho6wpQRRUpdX8K6KISBJCYSlm1hq' # 替换为用户的secret_key region_info = "sh" # 替换为用户的region,例如 sh 表示华东园区, gz 表示华南园区, tj 表示华北园区 self.cos_client = CosClient(appid, secret_id, secret_key, region=region_info) self.cos_path = cos_path self.local_path = local_path def upload(self, path): """ cos_path:/news/jiemian/image/ :param path :return: """ counter = 0 url = '' while counter != 10: try: # 得到hash uploadName = path.replace('image\\', '') request = UploadFileRequest(u"crawler", self.cos_path + uploadName, self.local_path + path, insert_only=0) upload_file_ret = self.cos_client.upload_file(request) if upload_file_ret['code'] == 0: data = upload_file_ret['data'] or {} url = data['source_url'] print u'上传成功 ' + url else: print u'上传图片失败', upload_file_ret break except Exception as e: counter += 1 timer_util.sleep(10) return url
#!/usr/bin/python # -*- coding: UTF-8 -*- from qcloud_cos import CosClient from qcloud_cos import UploadFileRequest from qcloud_cos import UploadSliceFileRequest from qcloud_cos import UpdateFileRequest from qcloud_cos import UpdateFolderRequest from qcloud_cos import DelFileRequest from qcloud_cos import DelFolderRequest from qcloud_cos import CreateFolderRequest from qcloud_cos import StatFileRequest from qcloud_cos import StatFolderRequest from qcloud_cos import ListFolderRequest import logging import sys appid = 1253506722 secret_id = u'AKIDjKjx2vmhePxKldwwKaeVEl5C8vX8pAPS' secret_key = u'0rVxEKDuOr9JH5h8OnOn026zhT8bswpi' region_info = "tj" cos_client = CosClient(appid, secret_id, secret_key, region=region_info) request = UploadFileRequest(u"codingdie01", sys.argv[2].decode("utf-8"), sys.argv[1].decode("utf-8")) result = cos_client.upload_file(request) print result
class CosV4StorageService(storage_service.StorageService): def __init__(self, *args, **kwargs): appid = int(kwargs['appid']) region = kwargs['region'] accesskeyid = unicode(kwargs['accesskeyid']) accesskeysecret = unicode(kwargs['accesskeysecret']) bucket = unicode(kwargs['bucket']) if 'prefix_dir' in kwargs: self._prefix_dir = kwargs['prefix_dir'] else: self._prefix_dir = None self._cos_api = CosClient(appid, accesskeyid, accesskeysecret, region=region) self._bucket = bucket self._overwrite = kwargs[ 'overwrite'] == 'true' if 'overwrite' in kwargs else False self._max_retry = 20 def download(self, task, local_path): # self._oss_api.get_object_to_file(urllib.unquote(cos_path).encode('utf-8'), local_path) for i in range(20): logger.info("download file with rety {0}".format(i)) import os try: os.remove(task.key) except: pass req = DownloadFileRequest(self._bucket, task.key, local_path) ret = self._cos_api.download_file(req) # self._oss_api.get_object_to_file(task.key, local_path) logger.debug(str(ret)) if task.size is None: logger.info( "task's size is None, skip check file size on local") break from os import path if path.getsize(local_path) != int(task.size): logger.error( "Download Failed, size1: {size1}, size2: {size2}".format( size1=path.getsize(local_path), size2=task.size)) else: logger.info("Download Successfully, break") break else: raise IOError("Download Failed with 20 retry") def upload(self, task, local_path): cos_path = task.key if not cos_path.startswith('/'): cos_path = '/' + cos_path if self._prefix_dir: cos_path = self._prefix_dir + cos_path if isinstance(local_path, unicode): local_path.encode('utf-8') insert_only = 0 if self._overwrite else 1 for i in range(10): try: upload_request = UploadFileRequest(self._bucket, unicode(cos_path), local_path, insert_only=insert_only) upload_file_ret = self._cos_api.upload_file(upload_request) if upload_file_ret[u'code'] != 0: logger.warn("upload failed:" + str(upload_file_ret)) raise OSError("UploadError: " + str(upload_file_ret)) else: break except: pass else: raise IOError("upload failed") def list(self): if self._prefix_dir is None: for i in self.dfs('/'): yield i else: for i in self.dfs(self._prefix_dir): yield i def dfs(self, path): print "DFS: {path}".format(path=to_utf8(path)) _finish = False _context = u'' max_retry = self._max_retry path = to_unicode(path) while not _finish: request = ListFolderRequest(bucket_name=self._bucket, cos_path=path, context=_context) ret = self._cos_api.list_folder(request) if ret['code'] != 0: max_retry -= 1 else: _finish = ret['data']['listover'] _context = ret['data']['context'] for item in ret['data']['infos']: if 'filelen' in item: try: key = "{prefix}{filename}".format( prefix=path, filename=item['name']) yield Task(key, item['filelen'], None) except: pass else: _sub_dir = "{prefix}{filename}".format( prefix=path.encode('utf-8'), filename=item['name'].encode('utf-8')) for i in self.dfs(_sub_dir): yield i if max_retry == 0: _finish = True def exists(self, task): _path = task.key _size = task.size if not _path.startswith('/'): _path = '/' + _path logger = getLogger(__name__) # logger.info("func: exists: " + str(_path)) if self._prefix_dir: _path = self._prefix_dir + _path if isinstance(_path, str): _path = _path.decode('utf-8') request = StatFileRequest(self._bucket, _path) ret = self._cos_api.stat_file(request) logger.debug("ret: " + str(ret)) # import json # v = json.loads(ret) if ret['code'] != 0: # logger.warn("error code: " + str(ret['code'])) return False if ret['data']['filelen'] != ret['data']['filesize']: logger.warn( "file is broken, filelen: {len}, filesize: {size}".format( len=ret['data']['filelen'], size=ret['data']['filesize'])) return False elif _size is not None and ret['data']['filelen'] != _size: return False elif ret['data']['filelen'] != _size: return False return True
def cos_demo(): # 设置用户属性, 包括appid, secret_id和secret_key # 这些属性可以在cos控制台获取(https://console.qcloud.com/cos) appid = 111 # 替换为用户的appid secret_id = u'' # 替换为用户的secret_id secret_key = u'' # 替换为用户的secret_key region = "shanghai" # # 替换为用户的region,目前可以为 shanghai/guangzhou cos_client = CosClient(appid, secret_id, secret_key, region) # 设置要操作的bucket bucket = u'' ############################################################################ # 文件操作 # ############################################################################ # 1. 上传文件(默认不覆盖) # 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt # 默认不覆盖, 如果cos上文件存在,则会返回错误 request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_1.txt') upload_file_ret = cos_client.upload_file(request) logger.info("upload file, return message: " + str(upload_file_ret)) # 分片上传大文件 request = UploadSliceFileRequest(bucket, u'/sample_bigfile.txt', u'local_bigfile.txt') ret = cos_client.upload_slice_file(request) logger.info("slice upload, return message: " + str(ret)) # 2. 上传文件(覆盖文件) # 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt with open('local_file_2.txt', 'w') as f: f.write("hello world2") request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_2.txt') request.set_insert_only(0) # 设置允许覆盖 upload_file_ret = cos_client.upload_file(request) logger.info('overwrite file, return message:' + str(upload_file_ret)) # 3. 获取文件属性 request = StatFileRequest(bucket, u'/sample_file.txt') stat_file_ret = cos_client.stat_file(request) logger.info('stat file, return message: ' + str(stat_file_ret)) # 4. 更新文件属性 request = UpdateFileRequest(bucket, u'/sample_file.txt') request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性 request.set_authority(u'eWRPrivate') # 设置文件的权限 request.set_cache_control(u'cache_xxx') # 设置Cache-Control request.set_content_type(u'application/text') # 设置Content-Type request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition request.set_content_language(u'english') # 设置Content-Language request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性 request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性 update_file_ret = cos_client.update_file(request) logger.info('update file, return message: ' + str(update_file_ret)) # 5. 更新后再次获取文件属性 request = StatFileRequest(bucket, u'/sample_file.txt') stat_file_ret = cos_client.stat_file(request) logger.info('stat file, return message: ' + str(stat_file_ret)) ############################################################################ # 目录操作 # ############################################################################ # 1. 生成目录, 目录名为sample_folder request = CreateFolderRequest(bucket, u'/sample_folder/') create_folder_ret = cos_client.create_folder(request) logger.info('create folder ret:' + str(create_folder_ret)) # 2. 更新目录的biz_attr属性 request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录') update_folder_ret = cos_client.update_folder(request) logger.info('update folder ret:' + str(update_folder_ret)) # 3. 获取目录属性 request = StatFolderRequest(bucket, u'/sample_folder/') stat_folder_ret = cos_client.stat_folder(request) logger.info("stat folder, return message: " + str(stat_folder_ret)) # 4. list目录, 获取目录下的成员 request = ListFolderRequest(bucket, u'/sample_folder/') list_folder_ret = cos_client.list_folder(request) logger.info("list folder, return message: " + str(list_folder_ret)) # 5. 删除目录 request = DelFolderRequest(bucket, u'/sample_folder/') delete_folder_ret = cos_client.del_folder(request) logger.info("delete folder, return message: " + str(delete_folder_ret))
def cos_demo(): # 设置用户属性, 包括appid, secret_id和secret_key # 这些属性可以在cos控制台获取(https://console.qcloud.com/cos) appid = 100000 # 替换为用户的appid secret_id = u'xxxxxxxx' # 替换为用户的secret_id secret_key = u'xxxxxxx' # 替换为用户的secret_key cos_client = CosClient(appid, secret_id, secret_key) # 设置要操作的bucket bucket = u'mybucket' ############################################################################ # 文件操作 # ############################################################################ # 1. 上传文件(默认不覆盖) # 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt # 默认不覆盖, 如果cos上文件存在,则会返回错误 request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_1.txt') upload_file_ret = cos_client.upload_file(request) print 'upload file ret:', repr(upload_file_ret) # 2. 上传文件(覆盖文件) # 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_2.txt') request.set_insert_only(0) # 设置允许覆盖 upload_file_ret = cos_client.upload_file(request) print 'overwrite file ret:', repr(upload_file_ret) # 3. 获取文件属性 request = StatFileRequest(bucket, u'/sample_file.txt') stat_file_ret = cos_client.stat_file(request) print 'stat file ret:', repr(stat_file_ret) # 4. 更新文件属性 request = UpdateFileRequest(bucket, u'/sample_file.txt') request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性 request.set_authority(u'eWRPrivate') # 设置文件的权限 request.set_cache_control(u'cache_xxx') # 设置Cache-Control request.set_content_type(u'application/text') # 设置Content-Type request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition request.set_content_language(u'english') # 设置Content-Language request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性 request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性 update_file_ret = cos_client.update_file(request) print 'update file ret:', repr(update_file_ret) # 5. 更新后再次获取文件属性 request = StatFileRequest(bucket, u'/sample_file.txt') stat_file_ret = cos_client.stat_file(request) print 'stat file ret:', repr(stat_file_ret) # 6. 移动文件, 将sample_file.txt移动位sample_file_move.txt request = MoveFileRequest(bucket, u'/sample_file.txt', u'/sample_file_move.txt') stat_file_ret = cos_client.move_file(request) print 'move file ret:', repr(stat_file_ret) # 7. 删除文件 request = DelFileRequest(bucket, u'/sample_file_move.txt') del_ret = cos_client.del_file(request) print 'del file ret:', repr(del_ret) ############################################################################ # 目录操作 # ############################################################################ # 1. 生成目录, 目录名为sample_folder request = CreateFolderRequest(bucket, u'/sample_folder/') create_folder_ret = cos_client.create_folder(request) print 'create folder ret:', create_folder_ret # 2. 更新目录的biz_attr属性 request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录') update_folder_ret = cos_client.update_folder(request) print 'update folder ret:', repr(update_folder_ret) # 3. 获取目录属性 request = StatFolderRequest(bucket, u'/sample_folder/') stat_folder_ret = cos_client.stat_folder(request) print 'stat folder ret:', repr(stat_folder_ret) # 4. list目录, 获取目录下的成员 request = ListFolderRequest(bucket, u'/sample_folder/') list_folder_ret = cos_client.list_folder(request) print 'list folder ret:', repr(list_folder_ret) # 5. 删除目录 request = DelFolderRequest(bucket, u'/sample_folder/') delete_folder_ret = cos_client.del_folder(request) print 'delete folder ret:', repr(delete_folder_ret)
#!/usr/bin/env python # coding=utf-8 from qcloud_cos import CosClient from qcloud_cos import UploadFileRequest appid = 100000 # 替换为你的appid secret_id = u'' # 替换为你的secret_id secret_key = u'' # 替换为你的secret_key bucket = u'' # 替换为你要操作的bucket名 region = "tj" # 替换为该bucket所属的地区代码, tj/sh/gz/sgp cos_client = CosClient(appid, secret_id, secret_key, region) ################################## # 文件操作 # ################################## # 上传文件(默认不允许覆盖) # 将本地的 upload_example.txt 上传到 bucket 的根分区下,并命名为 upload_sample_python.txt request = UploadFileRequest(bucket, u'/upload_sample_python.txt', u'/data/upload/upload_example.txt') upload_file_response = cos_client.upload_file(request) print upload_file_response
class UploadUtil(object): def __init__(self, cos_path, local_path): appid = 1251316472 # 替换为用户的appid secret_id = u'AKID3atJdSnNR1Bwf1l4HnnYaSMzpBArC4A5' # 替换为用户的secret_id secret_key = u'I2J7ho6wpQRRUpdX8K6KISBJCYSlm1hq' # 替换为用户的secret_key region_info = "sh" # 替换为用户的region,例如 sh 表示华东园区, gz 表示华南园区, tj 表示华北园区 self.cos_client = CosClient(appid, secret_id, secret_key, region=region_info) self.cos_path = cos_path self.local_path = local_path def uploadList(self, listFile): """ cos_path:/news/jiemian/image/ :param listFile: [{url,path},{}] :return: """ results = [] for item in listFile: # 得到hash path = item['path'] uploadName = path.replace(u'full/', '') request = UploadFileRequest(u"crawler", self.cos_path + uploadName, self.local_path + path, insert_only=0) upload_file_ret = self.cos_client.upload_file(request) if upload_file_ret['code'] == 0: data = upload_file_ret['data'] or {} url = data['source_url'] results.append(url) print u'upload successfully', url else: print u'fail to uplaod ', upload_file_ret return results def upload(self, path): """ cos_path:/news/jiemian/image/ :param path :return: """ counter = 0 url = '' while counter != 10: try: # 得到hash uploadName = path.replace('full/', '') request = UploadFileRequest(u"crawler", self.cos_path + uploadName, self.local_path + path, insert_only=0) upload_file_ret = self.cos_client.upload_file(request) if upload_file_ret['code'] == 0: data = upload_file_ret['data'] or {} url = data['source_url'] print u'上传成功', url else: print u'上传图片失败', upload_file_ret break except Exception as e: counter += 1 TimerUtil.sleep(10) return url
from qcloud_cos import UploadFileRequest from qcloud_cos import CosClient import sys if __name__ == '__main__': if len(sys.argv) < 3: print "argv error" bucket = u'koprvhdix117' app_id = 10038234 secret_id = u'AKIDkTQGMuCeJvtTTlqg911BfF393ghumqHp' secret_key = u'ZE1uBa6jfbsB0vVyfbWhw5JuZKPwaEwh' cos_client = CosClient(app_id, secret_id, secret_key) if sys.argv[1] == "upload": cos_path = unicode(sys.argv[2]) local_path = unicode(sys.argv[3]) request = UploadFileRequest(bucket, cos_path, local_path) upload_file_ret = cos_client.upload_file(request) print upload_file_ret elif sys.argv[1] == "delete": cos_path = unicode(sys.argv[2]) request = DelFileRequest(bucket, cos_path) delete_file_ret = cos_client.del_file(request) print delete_file_ret else: print "argv error"
def cos_demo(): # 设置用户属性, 包括appid, secret_id和secret_key # 这些属性可以在cos控制台获取(https://console.qcloud.com/cos) appid = 100000 # 替换为用户的appid secret_id = u"xxxxxxxx" # 替换为用户的secret_id secret_key = u"xxxxxxx" # 替换为用户的secret_key cos_client = CosClient(appid, secret_id, secret_key) # 设置要操作的bucket bucket = u"mybucket" ############################################################################ # 文件操作 # ############################################################################ # 1. 上传文件(默认不覆盖) # 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt # 默认不覆盖, 如果cos上文件存在,则会返回错误 request = UploadFileRequest(bucket, u"/sample_file.txt", u"local_file_1.txt") upload_file_ret = cos_client.upload_file(request) print "upload file ret:", repr(upload_file_ret) # 2. 上传文件(覆盖文件) # 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt request = UploadFileRequest(bucket, u"/sample_file.txt", u"local_file_2.txt") request.set_insert_only(0) # 设置允许覆盖 upload_file_ret = cos_client.upload_file(request) print "overwrite file ret:", repr(upload_file_ret) # 3. 获取文件属性 request = StatFileRequest(bucket, u"/sample_file.txt") stat_file_ret = cos_client.stat_file(request) print "stat file ret:", repr(stat_file_ret) # 4. 更新文件属性 request = UpdateFileRequest(bucket, u"/sample_file.txt") request.set_biz_attr(u"这是个demo文件") # 设置文件biz_attr属性 request.set_authority(u"eWRPrivate") # 设置文件的权限 request.set_cache_control(u"cache_xxx") # 设置Cache-Control request.set_content_type(u"application/text") # 设置Content-Type request.set_content_disposition(u"ccccxxx.txt") # 设置Content-Disposition request.set_content_language(u"english") # 设置Content-Language request.set_x_cos_meta(u"x-cos-meta-xxx", u"xxx") # 设置自定义的x-cos-meta-属性 request.set_x_cos_meta(u"x-cos-meta-yyy", u"yyy") # 设置自定义的x-cos-meta-属性 update_file_ret = cos_client.update_file(request) print "update file ret:", repr(update_file_ret) # 5. 更新后再次获取文件属性 request = StatFileRequest(bucket, u"/sample_file.txt") stat_file_ret = cos_client.stat_file(request) print "stat file ret:", repr(stat_file_ret) # 6. 移动文件, 将sample_file.txt移动位sample_file_move.txt request = MoveFileRequest(bucket, u"/sample_file.txt", u"/sample_file_move.txt") stat_file_ret = cos_client.move_file(request) print "move file ret:", repr(stat_file_ret) # 7. 删除文件 request = DelFileRequest(bucket, u"/sample_file_move.txt") del_ret = cos_client.del_file(request) print "del file ret:", repr(del_ret) ############################################################################ # 目录操作 # ############################################################################ # 1. 生成目录, 目录名为sample_folder request = CreateFolderRequest(bucket, u"/sample_folder/") create_folder_ret = cos_client.create_folder(request) print "create folder ret:", create_folder_ret # 2. 更新目录的biz_attr属性 request = UpdateFolderRequest(bucket, u"/sample_folder/", u"这是一个测试目录") update_folder_ret = cos_client.update_folder(request) print "update folder ret:", repr(update_folder_ret) # 3. 获取目录属性 request = StatFolderRequest(bucket, u"/sample_folder/") stat_folder_ret = cos_client.stat_folder(request) print "stat folder ret:", repr(stat_folder_ret) # 4. list目录, 获取目录下的成员 request = ListFolderRequest(bucket, u"/sample_folder/") list_folder_ret = cos_client.list_folder(request) print "list folder ret:", repr(list_folder_ret) # 5. 删除目录 request = DelFolderRequest(bucket, u"/sample_folder/") delete_folder_ret = cos_client.del_folder(request) print "delete folder ret:", repr(delete_folder_ret)
class CosFS(object): def __init__(self, appid, secret_id, secret_key, bucket, region = None): self.appid = appid self.secret_id = secret_id self.secret_key = secret_key self.bucket = bucket if region: self.cos_client = CosClient(appid, secret_id, secret_key, region=region) else: self.cos_client = CosClient(appid, secret_id, secret_key) def list_dir(self, path=u'/'): path = to_unicode(path) prefix = u'' if path.endswith(u'*'): prefix = os.path.basename(path)[:-1] path = os.path.dirname(path) if not path.endswith(u'/'): path += u'/' context=u'' arr_data = [] while True: request = ListFolderRequest(self.bucket, path, prefix=prefix, context=context) result = self.cos_client.list_folder(request) if result['code'] != 0: raise CosFSException(result['code'], result['message'] + ': ' + path) data = result['data'] arr_data.append(data) has_more = False if 'has_more' in data: has_more = data['has_more'] if 'listover' in data: #cos v4 has_more = not data['listover'] if has_more: context = data['context'] else: break dataset = {'dircount': 0, 'filecount': 0, 'infos': []} for data in arr_data: if 'filecount' not in data: data['filecount'] = 0 data['dircount'] = 0 for entry in data['infos']: if self.isFile(entry): data['filecount'] += 1 else: entry['name'] = entry['name'].rstrip(u'/') data['dircount'] += 1 dataset['dircount'] += data['dircount'] dataset['filecount'] += data['filecount'] dataset['infos'] += data['infos'] return dataset def ls(self, path=u'/', detail=False, recursive=False): content = self.list_dir(path) if recursive: print '%s:' % (path.encode('utf-8')) for entry in content['infos']: isFile = self.isFile(entry) name = entry['name'].encode('utf-8') if detail: ctime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(entry['ctime']))) print './%s%s: [size:%d] [created_at:%s]' % (name, '' if isFile else '/', entry['filesize'] if isFile else 0, ctime) else: print name + ('' if isFile else '/') if recursive and content['dircount'] > 0: print '' for info in content['infos']: if 'sha' not in info: self.ls(path.rstrip(u'/') + u'/' + info['name'], detail, recursive) print '' def mv(self, src, dest, overwrite=False): src = to_unicode(src) dest = to_unicode(dest) if dest.endswith(u'/'): dest += os.path.basename(src) request = MoveFileRequest(self.bucket, src, dest) result = self.cos_client.move_file(request) if result['code'] != 0: raise CosFSException(result['code'], result['message']) def download(self, remote, local, overwrite=False): local = to_unicode(local) remote = to_unicode(remote) if local == u'.': local += u'/' if local.endswith(u'/'): local += os.path.basename(remote) if os.path.exists(local): if overwrite: os.unlink(local) else: raise CosFSException(-1, 'local file %s exists' % local) fileattr = self.stat(remote) url = fileattr['source_url'] + '?sign=' + fileattr['sign'] download_file(url, local) def cat(self, path): fileattr = self.stat(to_unicode(path)) url = fileattr['source_url'] + '?sign=' + fileattr['sign'] download_file(url, '/dev/stdout') def upload(self, local, remote, overwrite=False, silent=False): local = to_unicode(local) remote = to_unicode(remote) if remote.endswith(u'/'): remote += os.path.basename(local) request = UploadFileRequest(self.bucket, remote, local) if overwrite: request.set_insert_only(0) result = self.cos_client.upload_file(request) if result['code'] != 0: if result['code'] == CODE_SAME_FILE: if not silent: print >>sys.stderr, "skipped: same file on COS" return if overwrite and result['code'] == CODE_ERR_OFF_GOBACK: print >>sys.stderr, "fix tencent bug: remove remote file when ErrOffGoBack occurs" """ 上传出现这个4024的错误之后,覆盖(insertonly=0)参数也不能成功,只能删除文件后重新上传 建议您将分片大小改为1M 分片上传之间sleep 100ms 出现错误的概率会小很多 抱歉,这个问题暂时无法彻底解决,给您带来了不便。 """ self.rm(remote) raise CosFSException(result['code'], result['message']) def cp(self, src, dest, overwrite=False): if src.startswith('cos:') and dest.startswith('cos:'): #move in cos raise CosFSException(-1, "not supported yet, use mv please") elif src.startswith('cos:'): #download self.download(src[4:], dest, overwrite) elif dest.startswith('cos:'): #upload self.upload(src, dest[4:], overwrite) else: raise CosFSException(-1, "at least one of src/dest should start with `cos:`") def cpdir(self, src, dest, conflict=CONFLICT_ERROR): begin_at = time.time() src = to_unicode(src) dest = to_unicode(dest) if src.startswith(u'cos:') and dest.startswith(u'cos:'): raise CosFSException(-1, "not supported") elif src.startswith(u'cos:'): #download self.downloadDir(src[4:], dest, conflict) elif dest.startswith(u'cos:'): #upload self.uploadDir(src, dest[4:], conflict) else: raise CosFSException(-1, "at least one of src/dest should start with `cos:`") usage = time.time() - begin_at #print '[Time Usage: %.6f]' % usage #将[cos上remote目录里]的内容下载到[本地local目录里],如果local目录不存在,会被创建 def downloadDir(self, remote, local, conflict): remote = remote.rstrip(u'/') local = local.rstrip(u'/') file_queue = Queue.Queue() def walk(path = u'/', level = 0): print '[mkdir] ' + ' ' * level + local + path localMkdir(local + path) content = self.list_dir(remote + path) for entry in content['infos']: name = path + entry['name'].encode('utf-8') if self.isFile(entry): print '[copy] ' + ' ' * level + local + name overwrite = conflict == CONFLICT_OVERWRITE file_queue.put([self.download, (remote + name, local + name, overwrite)]) else: walk(name + '/', level + 1) walk() CosThread.execute(file_queue) print >>sys.stderr, "[download finished]" def uploadDir(self, local, remote, conflict): if not remote.endswith(u'/'): remote += u'/' if not local.endswith(u'/'): remote += os.path.basename(local) + u'/' local = os.path.abspath(local) + u'/' if not os.path.isdir(local): raise CosFSException(-1, "please specify a local directory") retry(self.mkdir, remote) def uploadFile(localfile, remotefile): try: print >>sys.stderr, '[uploadFile] %s => %s' % (localfile, remotefile) self.upload(localfile, remotefile, silent=True) print >>sys.stderr, 'done: new' except Exception, e: if e[0] == CODE_EXISTED: #ERROR_CMD_COS_FILE_EXIST if conflict == CONFLICT_SKIP: print >>sys.stderr, 'skipped: existed' elif conflict == CONFLICT_OVERWRITE: self.upload(localfile, remotefile, overwrite=True) print >>sys.stderr, 'done: overwrite' else: raise return raise file_queue = Queue.Queue() def process_dir(arg, dirname, filelist): dir_suffix = dirname[len(local):] remote_dir = remote + dir_suffix file_queue.put([self.mkdir, (remote_dir,)]) print >>sys.stderr, '[doUpload] queue for mkdir %s' % remote_dir for filename in filelist: localfile = dirname.rstrip(u'/') + u'/' + filename if os.path.isdir(localfile): continue if os.path.islink(localfile): print >>sys.stderr, '[doUpload] skip symlink %s' % (localfile) continue remotefile = remote_dir.rstrip(u'/') + u'/' + filename file_queue.put([uploadFile, (localfile, remotefile)]) print >>sys.stderr, '[doUpload] queue for upload file %s ...' % remotefile os.path.walk(local, process_dir, None) CosThread.execute(file_queue) print >>sys.stderr, "[upload finished]"
class CosV4StorageService(storage_service.StorageService): def __init__(self, *args, **kwargs): appid = int(kwargs['appid']) region = kwargs['region'] accesskeyid = unicode(kwargs['accesskeyid']) accesskeysecret = unicode(kwargs['accesskeysecret']) bucket = unicode(kwargs['bucket']) if 'prefix_dir' in kwargs: self._prefix_dir = kwargs['prefix_dir'] else: self._prefix_dir = None # filelist存在时, prefix_dir不生效 if 'filelist' in kwargs: self._filelist = kwargs['filelist'] self._prefix_dir = None else: self._filelist = None if 'sync_files' in kwargs: self._sync_files_dir = kwargs['sync_files'] else: self._sync_files_dir = '/tmp/workspace/sync_files' self._cos_api = CosClient(appid, accesskeyid, accesskeysecret, region=region) self._bucket = bucket self._overwrite = kwargs['overwrite'] == 'true' if 'overwrite' in kwargs else False self._max_retry = 20 def download(self, task, local_path): return self._download(task, local_path, False) def _download(self, task, local_path, is_ignore_size): for i in range(20): logger.info("download file with rety {0}".format(i)) import os try: os.remove(task.key) except: pass req = DownloadFileRequest(self._bucket, task.key, local_path) ret = self._cos_api.download_file(req) logger.debug(str(ret)) if ret['code'] != 0: logger.error("error code: " + str(ret['code'])) raise IOError("Download Failed, ret=" + str(ret)) if is_ignore_size: logger.info("Download Successfully, break") break if task.size is None: logger.info("task's size is None, skip check file size on local") break from os import path task_size = 0 if isinstance(task.size, int): task_size = task.size else: task_size = int(task.size['filelen']) if path.getsize(local_path) != task_size: logger.error("Download Failed, size1: {size1}, size2: {size2}".format(size1=path.getsize(local_path), size2=task_size)) else: logger.info("Download Successfully, break") break else: raise IOError("Download Failed with 20 retry") def upload(self, task, local_path): cos_path = task.key if not cos_path.startswith('/'): cos_path = '/' + cos_path if self._prefix_dir: cos_path = self._prefix_dir + cos_path if isinstance(local_path, unicode): local_path.encode('utf-8') insert_only = 0 if self._overwrite else 1 for i in range(10): try: upload_request = UploadFileRequest(self._bucket, unicode(cos_path), local_path, insert_only=insert_only) upload_file_ret = self._cos_api.upload_file(upload_request) if upload_file_ret[u'code'] != 0: logger.warn("upload failed:" + str(upload_file_ret)) raise OSError("UploadError: " + str(upload_file_ret)) else: break except: pass else: raise IOError("upload failed") def list(self, marker): if self._filelist is not None and len(self._filelist) > 0: filelist_path = self._sync_files_dir + '/' + os.path.basename(self._filelist) filelist_task = Task(self._filelist, None, None, None) self._download(filelist_task, filelist_path, True) with open(filelist_path) as f: line = f.readline() while line: if not line.startswith('/'): line = '/' + line task = self.get_task_by_key(line.strip()) yield task line = f.readline() else: if self._prefix_dir is None: for i in self.dfs('/'): yield i else: for i in self.dfs(self._prefix_dir): yield i def dfs(self, path): print "DFS: {path}".format(path=to_utf8(path)) _finish = False _context = u'' max_retry = self._max_retry path = to_unicode(path) while not _finish: request = ListFolderRequest(bucket_name=self._bucket, cos_path=path, context=_context) request.set_delimiter = '/' ret = self._cos_api.list_folder(request) logger.debug(str(ret)) if ret['code'] != 0: max_retry -= 1 else: _finish = ret['data']['listover'] _context = ret['data']['context'] for item in ret['data']['infos']: if 'filelen' in item: try: key = "{prefix}{filename}".format(prefix=path, filename=item['name']) logger.info("key=%s", key) yield Task(key, item, None, None) except: pass else: _sub_dir = "{prefix}{filename}".format(prefix=path.encode('utf-8'), filename=item['name'].encode('utf-8')) for i in self.dfs(_sub_dir): yield i if max_retry == 0: _finish = True def exists(self, task): _path = task.key _size = task.size if not _path.startswith('/'): _path = '/' + _path logger = getLogger(__name__) # logger.info("func: exists: " + str(_path)) if self._prefix_dir: _path = self._prefix_dir + _path if isinstance(_path, str): _path = _path.decode('utf-8') request = StatFileRequest(self._bucket, _path) ret = self._cos_api.stat_file(request) logger.debug("ret: " + str(ret)) # import json # v = json.loads(ret) if ret['code'] != 0: # logger.warn("error code: " + str(ret['code'])) return False if int(ret['data']['filelen']) != int(ret['data']['filesize']): logger.warn("file is broken, filelen: {len}, filesize: {size}".format( len=ret['data']['filelen'], size=ret['data']['filesize'] )) return False elif _size is not None and int(ret['data']['filelen']) != int(_size): return False return True def get_task_by_key(self, key): _path = key if not _path.startswith('/'): _path = '/' + _path logger = getLogger(__name__) if isinstance(_path, str): _path = _path.decode('utf-8') request = StatFileRequest(self._bucket, _path) ret = self._cos_api.stat_file(request) logger.info("ret: " + str(ret)) # import json # v = json.loads(ret) if ret['code'] != 0: logger.warn("get task by key error, key = {},error code: {}".format(key, str(ret['code']))) return Task(key, None, None, None) return Task(key, int(ret['data']['filesize']), None, None)