def upload_cos(file):
"""
CVM 上报 https 错误：取消 https 试试
"""
appid = 100000 # 根据自己的情况填写
secret_id = u'# 根据自己的情况填写'
secret_key = u'# 根据自己的情况填写'
region = '# 根据自己的情况填写'
bucket = u'# 根据自己的情况填写'
cos_client = CosClient(appid, secret_id, secret_key, region)
cos_path = u'/' + file
request = UploadFileRequest(bucket, cos_path, unicode(file))
request.set_insert_only(0)
start = time.time()
resp = cos_client.upload_file(request)
# assert resp['code'] == 0
if resp['code'] != 0:
return 0
end = time.time()
elapsed = (end - start) * 1000.0
print u'本次 COS 文件上传时间为 %0.3f ms' % elapsed
return elapsed
def update_packages(url_list):
cos_client = CosClient(app_id, secret_id, secret_key, region)
for package in url_list:
print package.filename
if package.status == 'need update':
print 'update'
elif package.status == 'not exist':
print 'downloading', package.filename
urllib.urlretrieve(package.url,
os.path.join('packages', 'CTS',
package.filename),
reporthook=report)
sys.stdout.flush()
else:
print 'skip', package.filename
request = StatFileRequest(bucket, unicode('/cts/' + package.filename))
stat_file_ret = cos_client.stat_file(request)
if stat_file_ret['code'] != 0:
file_path = os.path.join(os.path.dirname(__file__), 'packages',
'CTS', package.filename)
request = UploadFileRequest(bucket,
unicode('/cts/' + package.filename),
unicode(file_path))
upload_file_ret = cos_client.upload_file(request)
print upload_file_ret
def __init__(self, bucket, cos_path):
self.cos_client = CosClient(AudioRecord.appid, AudioRecord.secret_id,
AudioRecord.secret_key)
self.auth = Auth(self.cos_client.get_cred())
self.bucket = bucket
self.cos_path = cos_path
self.expired = 30
def __init__(self, *args, **kwargs):
appid = int(kwargs['appid'])
region = kwargs['region']
accesskeyid = unicode(kwargs['accesskeyid'])
accesskeysecret = unicode(kwargs['accesskeysecret'])
bucket = unicode(kwargs['bucket'])
if 'prefix_dir' in kwargs:
self._prefix_dir = kwargs['prefix_dir']
else:
self._prefix_dir = None
# filelist存在时, prefix_dir不生效
if 'filelist' in kwargs:
self._filelist = kwargs['filelist']
self._prefix_dir = None
else:
self._filelist = None
if 'sync_files' in kwargs:
self._sync_files_dir = kwargs['sync_files']
else:
self._sync_files_dir = '/tmp/workspace/sync_files'
self._cos_api = CosClient(appid, accesskeyid, accesskeysecret, region=region)
self._bucket = bucket
self._overwrite = kwargs['overwrite'] == 'true' if 'overwrite' in kwargs else False
self._max_retry = 20
def __init__(self, cos_path, local_path):
appid = 1251316472 # 替换为用户的appid
secret_id = u'AKID3atJdSnNR1Bwf1l4HnnYaSMzpBArC4A5' # 替换为用户的secret_id
secret_key = u'I2J7ho6wpQRRUpdX8K6KISBJCYSlm1hq' # 替换为用户的secret_key
region_info = "sh" # 替换为用户的region，例如 sh 表示华东园区, gz 表示华南园区, tj 表示华北园区
self.cos_client = CosClient(appid, secret_id, secret_key, region=region_info)
self.cos_path = cos_path
self.local_path = local_path
def syncLocalToCOS(appid,
secret_id,
secret_key,
bucket,
region_info,
root,
subFolder='',
debug=1):
root = formatPath(root)
subFolder = formatPath(subFolder)
cos_client = CosClient(appid, secret_id, secret_key, region=region_info)
# check cos_client & bucket
list_folder_ret = cos_client.list_folder(ListFolderRequest(bucket, '/'))
if list_folder_ret['message'] != 'SUCCESS':
raise Exception(
'>>> Check your appid / secret_id / secret_key / bucket_name <<<')
# 读取本地需要更新的目录
localFilesDict, localEmptyFolders = readLocalFiles(root,
subFolder,
debug=debug)
# 读取cos上需要更新的目录
cosFilesDict, cosEmptyFolders = readCosFiles(cos_client,
bucket,
subFolder,
debug=debug)
# 比较文件修改时间，筛选出需要更新的文件，并且上传。
modifiedLocalFiles = filterModifiedLocalFiles(localFilesDict,
cosFilesDict,
debug=debug)
if modifiedLocalFiles:
start = datetime.now()
if debug: drawTitle('Uploading files')
for file in modifiedLocalFiles:
uploadToCos(cos_client, bucket, root, file)
if debug: print('---\n%s: %s' % ('Used', datetime.now() - start))
# 筛选出cos上有，但本地已经不存在的文件，删除COS上的文件。
extraCosFiles = filterExtraCosFiles(localFilesDict,
cosFilesDict,
debug=debug)
if extraCosFiles:
start = datetime.now()
if debug: drawTitle('Deleting COS files')
for file in extraCosFiles:
deleteCosFile(cos_client, bucket, file)
if debug: print('---\n%s: %s' % ('Used', datetime.now() - start))
# 同步空文件夹
syncEmptyFolders(cos_client,
bucket,
localEmptyFolders,
cosEmptyFolders,
debug=debug)
def __init__(self, appid, secret_id, secret_key, bucket, region = None):
self.appid = appid
self.secret_id = secret_id
self.secret_key = secret_key
self.bucket = bucket
if region:
self.cos_client = CosClient(appid, secret_id, secret_key, region=region)
else:
self.cos_client = CosClient(appid, secret_id, secret_key)
def cos_demo(mkdir, file_path):
appid = xxxxxx # 替换为用户的appid
secret_id = u'xxxxxxx' # 替换为用户的secret_id
secret_key = u'xxxxxxx' # 替换为用户的secret_key
region = "tj" # # 替换为用户的region，目前可以为 shanghai/guangzhou
cos_client = CosClient(appid, secret_id, secret_key, region)
bucket = u'xxxx'
request = UploadFileRequest(bucket, u'/' + mkdir + '/' + file_path + '',
u'' + file_path + '')
upload_file_ret = cos_client.upload_file(request)
print logger.info("upload file, return message: " + str(upload_file_ret))
def __init__(self, *args, **kwargs):
appid = int(kwargs['appid'])
region = kwargs['region']
accesskeyid = unicode(kwargs['accesskeyid'])
accesskeysecret = unicode(kwargs['accesskeysecret'])
bucket = unicode(kwargs['bucket'])
if 'prefix_dir' in kwargs:
self._prefix_dir = kwargs['prefix_dir']
else:
self._prefix_dir = None
self._cos_api = CosClient(appid, accesskeyid, accesskeysecret, region=region)
self._bucket = bucket
class Client(object):
cos = settings.TECENT_COS
appid, secret_id, secret_key = cos['APP_ID'], cos['SECRET_ID'], cos['SECRET_KEY']
region_info = 'tj'
baseurl = u'http://blog-1251509264.costj.myqcloud.com/'
def __init__(self):
self.bucket = u'blog'
self.client = CosClient(
self.appid,
self.secret_id,
self.secret_key,
region=self.region_info
)
def upload(self,filename):
sep = os.sep
#if sep in filename,meaning that the filename is a absolute file path.
if sep in filename:
filename = os.path.abspath(filename)
#the cos path is the name of file.
cos_path = os.path.basename(filename)
local_path = filename
else:
cos_path = filename
#get the absolute path of the file,if it is only a filename without any path .
local_path = os.path.abspath(filename)
try:
#sometimes,it needed to be encoded to utf-8
cos_path, local_path = unicode('/'+cos_path), unicode(local_path)
except:
cos_path = '/' + cos_path
request = UploadFileRequest(self.bucket,cos_path,local_path)
result = self.client.upload_file(request)
if result[u'code'] == 0:
return result['data']['source_url']
else:
return {'fail':result['message']}
def del_file(self,filename):
if os.path.isdir(filename):
return None
request = DelFileRequest(self.bucket,
unicode('/'+filename))
result = self.client.del_file(request)
print result
return result
def generate_html(packages):
page = markup.page()
page.init(title="CTS Packages", )
page.pre()
for package in packages:
page.a(package.filename, href=cos_info['cos_url'] + package.filename)
page.pre.close()
with open('index.html', 'w') as t:
t.write(str(page))
t.close()
cos_client = CosClient(app_id, secret_id, secret_key, region)
request = UploadFileRequest(bucket, unicode('/cts/index.html'),
unicode('index.html'))
upload_file_ret = cos_client.upload_file(request)
print upload_file_ret
def GetCosClient():
appid = 1253142785
secret_id = u'xxxx' #change to your own secret_id
secret_key = u'xxxx' #change to your own secret_key
region_info = "sh" #change to your own region
cos_client = CosClient(appid, secret_id, secret_key, region=region_info)
return cos_client
def __init__(self, *args, **kwargs):
appid = int(kwargs['appid'])
region = kwargs['region']
accesskeyid = unicode(kwargs['accesskeyid'])
accesskeysecret = unicode(kwargs['accesskeysecret'])
bucket = unicode(kwargs['bucket'])
if 'prefix_dir' in kwargs:
self._prefix_dir = kwargs['prefix_dir']
else:
self._prefix_dir = None
self._cos_api = CosClient(appid, accesskeyid, accesskeysecret, region=region)
self._bucket = bucket
self._overwrite = kwargs['overwrite'] == 'true' if 'overwrite' in kwargs else False
self._max_retry = 20
def sync(self):
local_file_dir_info = LocalFileDirInfo(self.config.get_local_path())
del_file_dict = self.db.file_stat_dict
del_dir_dict = self.db.dir_stat_dict
create_file_dict = {}
create_dir_dict = {}
for local_file_stat in local_file_dir_info.file_stat_list:
local_file_path = local_file_stat.file_path
if self.db.file_stat_dict.has_key(local_file_path):
if FileStat.compare(local_file_stat, self.db.file_stat_dict[local_file_path]) == 0:
del self.db.file_stat_dict[local_file_path]
else:
create_file_dict[local_file_path] = local_file_stat
else:
create_file_dict[local_file_path] = local_file_stat
for local_dir_stat in local_file_dir_info.dir_stat_list:
local_dir_path = local_dir_stat.file_path
if self.db.dir_stat_dict.has_key(local_dir_path):
del self.db.dir_stat_dict[local_dir_path]
else:
create_dir_dict[local_dir_path] = local_dir_stat
appid = int(self.config.get_appid())
secret_id = self.config.get_secret_id()
secret_key = self.config.get_secret_key()
bucket = self.config.get_bucket()
timeout = self.config.get_timeout()
cos_client = CosClient(appid, secret_id, secret_key)
cos_config = CosConfig()
cos_config.set_timeout(timeout)
if self.config.get_enable_https() == 1:
cos_config.enable_https()
cos_client.set_config(cos_config)
self._create_cos_target_dir(cos_client, bucket)
self._del_cos_sync_file(cos_client, bucket, del_file_dict)
self._del_cos_sync_dir(cos_client, bucket, del_dir_dict)
self._create_cos_sync_dir(cos_client, bucket, create_dir_dict)
self._create_cos_sync_file(cos_client, bucket, create_file_dict)
def __init__(self):
self.bucket = u'blog'
self.client = CosClient(
self.appid,
self.secret_id,
self.secret_key,
region=self.region_info
)
def delete_cos(file):
appid = 10000 # 根据自己的情况填写
secret_id = u'# 根据自己的情况填写'
secret_key = u'# 根据自己的情况填写'
region = '# 根据自己的情况填写'
bucket = u'# 根据自己的情况填写'
cos_client = CosClient(appid, secret_id, secret_key, region)
cos_path = u'/' + file
request = DelFileRequest(bucket, cos_path)
start = time.time()
resp = cos_client.del_file(request)
# assert resp['code'] == 0
end = time.time()
if resp['code'] != 0:
return 0
elapsed = (end - start) * 1000.0
print 'COS Delete File Time %0.3f ms' % elapsed
return elapsed
class AudioRecord:
appid = 10048692
secret_id = u'AKIDgknyBYkNKnpONeweTRwK9t6Nn0jn78yG'
secret_key = u'fBCXVJK1PpWPtYizb7vIGVMIJFm90GBa'
def __init__(self, bucket, cos_path):
self.cos_client = CosClient(AudioRecord.appid, AudioRecord.secret_id,
AudioRecord.secret_key)
self.auth = Auth(self.cos_client.get_cred())
self.bucket = bucket
self.cos_path = cos_path
self.expired = 30
def get_once_sign(self):
return self.auth.sign_once(self.bucket, self.cos_path)
def get_more_sign(self):
return self.auth.sign_more(self.bucket, self.cos_path, self.expired)
def get_down_sign(self):
return self.auth.sign_download(self.bucket, self.cos_path,
self.expired)
def DirIsExist(self):
return False
def CreateIntroduceDir(self):
#'/record/introduce/' + xxx
request = CreateFolderRequest(self.bucket, self.cos_path)
create_folder_ret = self.cos_client.create_folder(request)
result = json.loads(create_folder_ret)
return result
def DeleteFolderTest(self):
request = DelFolderRequest(self.bucket, self.cos_path)
delete_folder_ret = CosClient.del_folder(request)
result = json.loads(delete_folder_ret)
print 'delete folder ret:', repr(delete_folder_ret)
return result
def download_cos(file):
appid = 100000 # 根据自己的情况填写
secret_id = u'# 根据自己的情况填写'
secret_key = u'# 根据自己的情况填写'
region = '# 根据自己的情况填写'
bucket = u'# 根据自己的情况填写'
cos_client = CosClient(appid, secret_id, secret_key, region)
request = StatFileRequest(bucket, u'/' + file)
stat = cos_client.stat_file(request)
if stat['code'] != 0:
return 0
download_url = stat['data']['source_url'] # 使用外网直接访问 URL
start = time.time()
r = requests.get(download_url, timeout=60)
end = time.time()
if r.status_code != 200:
return 0
elapsed = (end - start) * 1000.0
print 'COS Download File Time %0.3f ms' % elapsed
return elapsed
def action_handler(event, context):
#Create CosClient to upload/download COS file
appid = 1253142785 # change to user's appid
secret_id = u'xxx' # change to user's secret_id
secret_key = u'xxx' # change to user's secret_key
region = u'sh' # change to user's region
cos_client = CosClient(appid, secret_id, secret_key, region)
#specify the source and destination bucket location
source_bucket = event['Records'][0]['cos']['cosBucket']['name']
source_bucket_file_key = '/' + event['Records'][0]['cos']['cosObject'][
'key'].split('/')[-1]
source_file_name = source_bucket_file_key.split('/')[-1].split('.')[0]
dest_bucket = u'output'
dest_bucket_file_key = u'/max_temperature_' + source_file_name
#specify the temp file location
source_file_tmp_path = u'/tmp/' + source_file_name
dest_file_temp_path = u'/tmp/max_temperature_' + source_file_name
#download the source file from cos bucket and take actions
download_ret = download_file(cos_client, source_bucket,
source_bucket_file_key, source_file_tmp_path)
if download_ret == 0:
dest_file_temp = open(dest_file_temp_path, 'w')
max_temp = -999.9
#find the maximum temperature
with gzip.open(source_file_tmp_path) as inputfile:
for line in inputfile:
temp = int(line[14:19]) / 10.0
if temp > max_temp:
max_temp = temp
#write the result to the temp file and upload to the cos bucket
dest_file_temp.write(source_file_name + ' ' + str(max_temp))
dest_file_temp.close()
upload_ret = upload_file(cos_client, dest_bucket, dest_bucket_file_key,
dest_file_temp_path)
return upload_ret
else:
return -1
class UploadUtil(object):
def __init__(self, cos_path, local_path):
appid = 1251316472 # 替换为用户的appid
secret_id = u'AKID3atJdSnNR1Bwf1l4HnnYaSMzpBArC4A5' # 替换为用户的secret_id
secret_key = u'I2J7ho6wpQRRUpdX8K6KISBJCYSlm1hq' # 替换为用户的secret_key
region_info = "sh" # 替换为用户的region，例如 sh 表示华东园区, gz 表示华南园区, tj 表示华北园区
self.cos_client = CosClient(appid,
secret_id,
secret_key,
region=region_info)
self.cos_path = cos_path
self.local_path = local_path
def upload(self, path):
"""
cos_path:/news/jiemian/image/
:param path
:return:
"""
counter = 0
url = ''
while counter != 10:
try:
# 得到hash
uploadName = path.replace('image\\', '')
request = UploadFileRequest(u"crawler",
self.cos_path + uploadName,
self.local_path + path,
insert_only=0)
upload_file_ret = self.cos_client.upload_file(request)
if upload_file_ret['code'] == 0:
data = upload_file_ret['data'] or {}
url = data['source_url']
print u'上传成功 ' + url
else:
print u'上传图片失败', upload_file_ret
break
except Exception as e:
counter += 1
timer_util.sleep(10)
return url
class CosV4StorageService(storage_service.StorageService):
def __init__(self, *args, **kwargs):
appid = int(kwargs['appid'])
region = kwargs['region']
accesskeyid = unicode(kwargs['accesskeyid'])
accesskeysecret = unicode(kwargs['accesskeysecret'])
bucket = unicode(kwargs['bucket'])
if 'prefix_dir' in kwargs:
self._prefix_dir = kwargs['prefix_dir']
else:
self._prefix_dir = None
self._cos_api = CosClient(appid, accesskeyid, accesskeysecret, region=region)
self._bucket = bucket
def download(self, cos_path, local_path):
raise NotImplementedError
def upload(self, cos_path, local_path):
if not cos_path.startswith('/'):
cos_path = '/' + cos_path
if self._prefix_dir:
cos_path = self._prefix_dir + cos_path
if isinstance(local_path, unicode):
local_path.encode('utf-8')
upload_request = UploadFileRequest(self._bucket, unicode(cos_path), local_path)
upload_file_ret = self._cos_api.upload_file(upload_request)
if upload_file_ret[u'code'] != 0:
raise OSError("UploadError: " + str(upload_file_ret))
def list(self):
raise NotImplementedError
def exists(self, _path):
raise NotImplementedError
class CosFS(object):
def __init__(self, appid, secret_id, secret_key, bucket, region = None):
self.appid = appid
self.secret_id = secret_id
self.secret_key = secret_key
self.bucket = bucket
if region:
self.cos_client = CosClient(appid, secret_id, secret_key, region=region)
else:
self.cos_client = CosClient(appid, secret_id, secret_key)
def list_dir(self, path=u'/'):
path = to_unicode(path)
prefix = u''
if path.endswith(u'*'):
prefix = os.path.basename(path)[:-1]
path = os.path.dirname(path)
if not path.endswith(u'/'):
path += u'/'
context=u''
arr_data = []
while True:
request = ListFolderRequest(self.bucket, path, prefix=prefix, context=context)
result = self.cos_client.list_folder(request)
if result['code'] != 0:
raise CosFSException(result['code'], result['message'] + ': ' + path)
data = result['data']
arr_data.append(data)
has_more = False
if 'has_more' in data:
has_more = data['has_more']
if 'listover' in data: #cos v4
has_more = not data['listover']
if has_more:
context = data['context']
else:
break
dataset = {'dircount': 0, 'filecount': 0, 'infos': []}
for data in arr_data:
if 'filecount' not in data:
data['filecount'] = 0
data['dircount'] = 0
for entry in data['infos']:
if self.isFile(entry):
data['filecount'] += 1
else:
entry['name'] = entry['name'].rstrip(u'/')
data['dircount'] += 1
dataset['dircount'] += data['dircount']
dataset['filecount'] += data['filecount']
dataset['infos'] += data['infos']
return dataset
def ls(self, path=u'/', detail=False, recursive=False):
content = self.list_dir(path)
if recursive:
print '%s:' % (path.encode('utf-8'))
for entry in content['infos']:
isFile = self.isFile(entry)
name = entry['name'].encode('utf-8')
if detail:
ctime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(entry['ctime'])))
print './%s%s: [size:%d] [created_at:%s]' % (name, '' if isFile else '/', entry['filesize'] if isFile else 0, ctime)
else:
print name + ('' if isFile else '/')
if recursive and content['dircount'] > 0:
print ''
for info in content['infos']:
if 'sha' not in info:
self.ls(path.rstrip(u'/') + u'/' + info['name'], detail, recursive)
print ''
def mv(self, src, dest, overwrite=False):
src = to_unicode(src)
dest = to_unicode(dest)
if dest.endswith(u'/'):
dest += os.path.basename(src)
request = MoveFileRequest(self.bucket, src, dest)
result = self.cos_client.move_file(request)
if result['code'] != 0:
raise CosFSException(result['code'], result['message'])
def download(self, remote, local, overwrite=False):
local = to_unicode(local)
remote = to_unicode(remote)
if local == u'.':
local += u'/'
if local.endswith(u'/'):
local += os.path.basename(remote)
if os.path.exists(local):
if overwrite:
os.unlink(local)
else:
raise CosFSException(-1, 'local file %s exists' % local)
fileattr = self.stat(remote)
url = fileattr['source_url'] + '?sign=' + fileattr['sign']
download_file(url, local)
def cat(self, path):
fileattr = self.stat(to_unicode(path))
url = fileattr['source_url'] + '?sign=' + fileattr['sign']
download_file(url, '/dev/stdout')
def upload(self, local, remote, overwrite=False, silent=False):
local = to_unicode(local)
remote = to_unicode(remote)
if remote.endswith(u'/'):
remote += os.path.basename(local)
request = UploadFileRequest(self.bucket, remote, local)
if overwrite:
request.set_insert_only(0)
result = self.cos_client.upload_file(request)
if result['code'] != 0:
if result['code'] == CODE_SAME_FILE:
if not silent:
print >>sys.stderr, "skipped: same file on COS"
return
if overwrite and result['code'] == CODE_ERR_OFF_GOBACK:
print >>sys.stderr, "fix tencent bug: remove remote file when ErrOffGoBack occurs"
"""
上传出现这个4024的错误之后，覆盖（insertonly=0）参数也不能成功，只能删除文件后重新上传
建议您将分片大小改为1M 分片上传之间sleep 100ms 出现错误的概率会小很多
抱歉，这个问题暂时无法彻底解决，给您带来了不便。
"""
self.rm(remote)
raise CosFSException(result['code'], result['message'])
def cp(self, src, dest, overwrite=False):
if src.startswith('cos:') and dest.startswith('cos:'): #move in cos
raise CosFSException(-1, "not supported yet, use mv please")
elif src.startswith('cos:'): #download
self.download(src[4:], dest, overwrite)
elif dest.startswith('cos:'): #upload
self.upload(src, dest[4:], overwrite)
else:
raise CosFSException(-1, "at least one of src/dest should start with `cos:`")
def cpdir(self, src, dest, conflict=CONFLICT_ERROR):
begin_at = time.time()
src = to_unicode(src)
dest = to_unicode(dest)
if src.startswith(u'cos:') and dest.startswith(u'cos:'):
raise CosFSException(-1, "not supported")
elif src.startswith(u'cos:'): #download
self.downloadDir(src[4:], dest, conflict)
elif dest.startswith(u'cos:'): #upload
self.uploadDir(src, dest[4:], conflict)
else:
raise CosFSException(-1, "at least one of src/dest should start with `cos:`")
usage = time.time() - begin_at
#print '[Time Usage: %.6f]' % usage
#将[cos上remote目录里]的内容下载到[本地local目录里]，如果local目录不存在，会被创建
def downloadDir(self, remote, local, conflict):
remote = remote.rstrip(u'/')
local = local.rstrip(u'/')
file_queue = Queue.Queue()
def walk(path = u'/', level = 0):
print '[mkdir] ' + ' ' * level + local + path
localMkdir(local + path)
content = self.list_dir(remote + path)
for entry in content['infos']:
name = path + entry['name'].encode('utf-8')
if self.isFile(entry):
print '[copy] ' + ' ' * level + local + name
overwrite = conflict == CONFLICT_OVERWRITE
file_queue.put([self.download, (remote + name, local + name, overwrite)])
else:
walk(name + '/', level + 1)
walk()
CosThread.execute(file_queue)
print >>sys.stderr, "[download finished]"
def uploadDir(self, local, remote, conflict):
if not remote.endswith(u'/'):
remote += u'/'
if not local.endswith(u'/'):
remote += os.path.basename(local) + u'/'
local = os.path.abspath(local) + u'/'
if not os.path.isdir(local):
raise CosFSException(-1, "please specify a local directory")
retry(self.mkdir, remote)
def uploadFile(localfile, remotefile):
try:
print >>sys.stderr, '[uploadFile] %s => %s' % (localfile, remotefile)
self.upload(localfile, remotefile, silent=True)
print >>sys.stderr, 'done: new'
except Exception, e:
if e[0] == CODE_EXISTED: #ERROR_CMD_COS_FILE_EXIST
if conflict == CONFLICT_SKIP:
print >>sys.stderr, 'skipped: existed'
elif conflict == CONFLICT_OVERWRITE:
self.upload(localfile, remotefile, overwrite=True)
print >>sys.stderr, 'done: overwrite'
else:
raise
return
raise
file_queue = Queue.Queue()
def process_dir(arg, dirname, filelist):
dir_suffix = dirname[len(local):]
remote_dir = remote + dir_suffix
file_queue.put([self.mkdir, (remote_dir,)])
print >>sys.stderr, '[doUpload] queue for mkdir %s' % remote_dir
for filename in filelist:
localfile = dirname.rstrip(u'/') + u'/' + filename
if os.path.isdir(localfile):
continue
if os.path.islink(localfile):
print >>sys.stderr, '[doUpload] skip symlink %s' % (localfile)
continue
remotefile = remote_dir.rstrip(u'/') + u'/' + filename
file_queue.put([uploadFile, (localfile, remotefile)])
print >>sys.stderr, '[doUpload] queue for upload file %s ...' % remotefile
os.path.walk(local, process_dir, None)
CosThread.execute(file_queue)
print >>sys.stderr, "[upload finished]"
def DeleteFolderTest(self):
request = DelFolderRequest(self.bucket, self.cos_path)
delete_folder_ret = CosClient.del_folder(request)
result = json.loads(delete_folder_ret)
print 'delete folder ret:', repr(delete_folder_ret)
return result
#!/usr/bin/env python
# coding=utf-8
from qcloud_cos import CosClient
from qcloud_cos import UploadFileRequest
appid = 100000 # 替换为你的appid
secret_id = u'' # 替换为你的secret_id
secret_key = u'' # 替换为你的secret_key
bucket = u'' # 替换为你要操作的bucket名
region = "tj" # 替换为该bucket所属的地区代码, tj/sh/gz/sgp
cos_client = CosClient(appid, secret_id, secret_key, region)
##################################
# 文件操作 #
##################################
# 上传文件(默认不允许覆盖)
# 将本地的 upload_example.txt 上传到 bucket 的根分区下,并命名为 upload_sample_python.txt
request = UploadFileRequest(bucket, u'/upload_sample_python.txt',
u'/data/upload/upload_example.txt')
upload_file_response = cos_client.upload_file(request)
print upload_file_response
class UploadUtil(object):
def __init__(self, cos_path, local_path):
appid = 1251316472 # 替换为用户的appid
secret_id = u'AKID3atJdSnNR1Bwf1l4HnnYaSMzpBArC4A5' # 替换为用户的secret_id
secret_key = u'I2J7ho6wpQRRUpdX8K6KISBJCYSlm1hq' # 替换为用户的secret_key
region_info = "sh" # 替换为用户的region，例如 sh 表示华东园区, gz 表示华南园区, tj 表示华北园区
self.cos_client = CosClient(appid,
secret_id,
secret_key,
region=region_info)
self.cos_path = cos_path
self.local_path = local_path
def uploadList(self, listFile):
"""
cos_path:/news/jiemian/image/
:param listFile: [{url,path},{}]
:return:
"""
results = []
for item in listFile:
# 得到hash
path = item['path']
uploadName = path.replace(u'full/', '')
request = UploadFileRequest(u"crawler",
self.cos_path + uploadName,
self.local_path + path,
insert_only=0)
upload_file_ret = self.cos_client.upload_file(request)
if upload_file_ret['code'] == 0:
data = upload_file_ret['data'] or {}
url = data['source_url']
results.append(url)
print u'upload successfully', url
else:
print u'fail to uplaod ', upload_file_ret
return results
def upload(self, path):
"""
cos_path:/news/jiemian/image/
:param path
:return:
"""
counter = 0
url = ''
while counter != 10:
try:
# 得到hash
uploadName = path.replace('full/', '')
request = UploadFileRequest(u"crawler",
self.cos_path + uploadName,
self.local_path + path,
insert_only=0)
upload_file_ret = self.cos_client.upload_file(request)
if upload_file_ret['code'] == 0:
data = upload_file_ret['data'] or {}
url = data['source_url']
print u'上传成功', url
else:
print u'上传图片失败', upload_file_ret
break
except Exception as e:
counter += 1
TimerUtil.sleep(10)
return url
request = CreateFolderRequest(self.bucket, self.cos_path)
create_folder_ret = self.cos_client.create_folder(request)
result = json.loads(create_folder_ret)
return result
def DeleteFolderTest(self):
request = DelFolderRequest(self.bucket, self.cos_path)
delete_folder_ret = CosClient.del_folder(request)
result = json.loads(delete_folder_ret)
print 'delete folder ret:', repr(delete_folder_ret)
return result
if __name__ == '__main__':
ar = CosClient(appid=AudioRecord.appid,
secret_id=AudioRecord.secret_id,
secret_key=AudioRecord.secret_key)
request = ListFolderRequest(u'porncheck', u'/1400011479/19700101/')
list_folder_ret = ar.list_folder(request)
print list_folder_ret
#request = DelFolderRequest(u'porncheck', u'/1400011479/19700101/')
#delete_folder_ret = ar.del_folder(request)
#print 'delete folder ret:', repr(delete_folder_ret)
#ar = AudioRecord('heydo','/record/introduce/102/')
#ar = AudioRecord('porncheck', '/1400011479/19700101/')
#print ar
#print ar.get_once_sign()
#print ar.DeleteFolderTest()
#print ar.get_once_sign()
#print ar.CreateIntroduceDir()
#cos_demo()
# coding:utf-8
from qcloud_cos import DelFileRequest
from qcloud_cos import UploadFileRequest
from qcloud_cos import CosClient
import sys
if __name__ == '__main__':
if len(sys.argv) < 3:
print "argv error"
bucket = u'koprvhdix117'
app_id = 10038234
secret_id = u'AKIDkTQGMuCeJvtTTlqg911BfF393ghumqHp'
secret_key = u'ZE1uBa6jfbsB0vVyfbWhw5JuZKPwaEwh'
cos_client = CosClient(app_id, secret_id, secret_key)
if sys.argv[1] == "upload":
cos_path = unicode(sys.argv[2])
local_path = unicode(sys.argv[3])
request = UploadFileRequest(bucket, cos_path, local_path)
upload_file_ret = cos_client.upload_file(request)
print upload_file_ret
elif sys.argv[1] == "delete":
cos_path = unicode(sys.argv[2])
request = DelFileRequest(bucket, cos_path)
delete_file_ret = cos_client.del_file(request)
print delete_file_ret
else:
def cos_demo():
# 设置用户属性, 包括appid, secret_id和secret_key
# 这些属性可以在cos控制台获取(https://console.qcloud.com/cos)
appid = 100000 # 替换为用户的appid
secret_id = u"xxxxxxxx" # 替换为用户的secret_id
secret_key = u"xxxxxxx" # 替换为用户的secret_key
cos_client = CosClient(appid, secret_id, secret_key)
# 设置要操作的bucket
bucket = u"mybucket"
############################################################################
# 文件操作 #
############################################################################
# 1. 上传文件(默认不覆盖)
# 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt
# 默认不覆盖, 如果cos上文件存在,则会返回错误
request = UploadFileRequest(bucket, u"/sample_file.txt", u"local_file_1.txt")
upload_file_ret = cos_client.upload_file(request)
print "upload file ret:", repr(upload_file_ret)
# 2. 上传文件(覆盖文件)
# 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt
request = UploadFileRequest(bucket, u"/sample_file.txt", u"local_file_2.txt")
request.set_insert_only(0) # 设置允许覆盖
upload_file_ret = cos_client.upload_file(request)
print "overwrite file ret:", repr(upload_file_ret)
# 3. 获取文件属性
request = StatFileRequest(bucket, u"/sample_file.txt")
stat_file_ret = cos_client.stat_file(request)
print "stat file ret:", repr(stat_file_ret)
# 4. 更新文件属性
request = UpdateFileRequest(bucket, u"/sample_file.txt")
request.set_biz_attr(u"这是个demo文件") # 设置文件biz_attr属性
request.set_authority(u"eWRPrivate") # 设置文件的权限
request.set_cache_control(u"cache_xxx") # 设置Cache-Control
request.set_content_type(u"application/text") # 设置Content-Type
request.set_content_disposition(u"ccccxxx.txt") # 设置Content-Disposition
request.set_content_language(u"english") # 设置Content-Language
request.set_x_cos_meta(u"x-cos-meta-xxx", u"xxx") # 设置自定义的x-cos-meta-属性
request.set_x_cos_meta(u"x-cos-meta-yyy", u"yyy") # 设置自定义的x-cos-meta-属性
update_file_ret = cos_client.update_file(request)
print "update file ret:", repr(update_file_ret)
# 5. 更新后再次获取文件属性
request = StatFileRequest(bucket, u"/sample_file.txt")
stat_file_ret = cos_client.stat_file(request)
print "stat file ret:", repr(stat_file_ret)
# 6. 移动文件, 将sample_file.txt移动位sample_file_move.txt
request = MoveFileRequest(bucket, u"/sample_file.txt", u"/sample_file_move.txt")
stat_file_ret = cos_client.move_file(request)
print "move file ret:", repr(stat_file_ret)
# 7. 删除文件
request = DelFileRequest(bucket, u"/sample_file_move.txt")
del_ret = cos_client.del_file(request)
print "del file ret:", repr(del_ret)
############################################################################
# 目录操作 #
############################################################################
# 1. 生成目录, 目录名为sample_folder
request = CreateFolderRequest(bucket, u"/sample_folder/")
create_folder_ret = cos_client.create_folder(request)
print "create folder ret:", create_folder_ret
# 2. 更新目录的biz_attr属性
request = UpdateFolderRequest(bucket, u"/sample_folder/", u"这是一个测试目录")
update_folder_ret = cos_client.update_folder(request)
print "update folder ret:", repr(update_folder_ret)
# 3. 获取目录属性
request = StatFolderRequest(bucket, u"/sample_folder/")
stat_folder_ret = cos_client.stat_folder(request)
print "stat folder ret:", repr(stat_folder_ret)
# 4. list目录, 获取目录下的成员
request = ListFolderRequest(bucket, u"/sample_folder/")
list_folder_ret = cos_client.list_folder(request)
print "list folder ret:", repr(list_folder_ret)
# 5. 删除目录
request = DelFolderRequest(bucket, u"/sample_folder/")
delete_folder_ret = cos_client.del_folder(request)
print "delete folder ret:", repr(delete_folder_ret)
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from qcloud_cos import CosClient
from qcloud_cos import UploadFileRequest
from qcloud_cos import UploadSliceFileRequest
from qcloud_cos import UpdateFileRequest
from qcloud_cos import UpdateFolderRequest
from qcloud_cos import DelFileRequest
from qcloud_cos import DelFolderRequest
from qcloud_cos import CreateFolderRequest
from qcloud_cos import StatFileRequest
from qcloud_cos import StatFolderRequest
from qcloud_cos import ListFolderRequest
import logging
import sys
appid = 1253506722
secret_id = u'AKIDjKjx2vmhePxKldwwKaeVEl5C8vX8pAPS'
secret_key = u'0rVxEKDuOr9JH5h8OnOn026zhT8bswpi'
region_info = "tj"
cos_client = CosClient(appid, secret_id, secret_key, region=region_info)
request = UploadFileRequest(u"codingdie01", sys.argv[2].decode("utf-8"),
sys.argv[1].decode("utf-8"))
result = cos_client.upload_file(request)
print result
class CosV4StorageService(storage_service.StorageService):
def __init__(self, *args, **kwargs):
appid = int(kwargs['appid'])
region = kwargs['region']
accesskeyid = unicode(kwargs['accesskeyid'])
accesskeysecret = unicode(kwargs['accesskeysecret'])
bucket = unicode(kwargs['bucket'])
if 'prefix_dir' in kwargs:
self._prefix_dir = kwargs['prefix_dir']
else:
self._prefix_dir = None
self._cos_api = CosClient(appid,
accesskeyid,
accesskeysecret,
region=region)
self._bucket = bucket
self._overwrite = kwargs[
'overwrite'] == 'true' if 'overwrite' in kwargs else False
self._max_retry = 20
def download(self, task, local_path):
# self._oss_api.get_object_to_file(urllib.unquote(cos_path).encode('utf-8'), local_path)
for i in range(20):
logger.info("download file with rety {0}".format(i))
import os
try:
os.remove(task.key)
except:
pass
req = DownloadFileRequest(self._bucket, task.key, local_path)
ret = self._cos_api.download_file(req)
# self._oss_api.get_object_to_file(task.key, local_path)
logger.debug(str(ret))
if task.size is None:
logger.info(
"task's size is None, skip check file size on local")
break
from os import path
if path.getsize(local_path) != int(task.size):
logger.error(
"Download Failed, size1: {size1}, size2: {size2}".format(
size1=path.getsize(local_path), size2=task.size))
else:
logger.info("Download Successfully, break")
break
else:
raise IOError("Download Failed with 20 retry")
def upload(self, task, local_path):
cos_path = task.key
if not cos_path.startswith('/'):
cos_path = '/' + cos_path
if self._prefix_dir:
cos_path = self._prefix_dir + cos_path
if isinstance(local_path, unicode):
local_path.encode('utf-8')
insert_only = 0 if self._overwrite else 1
for i in range(10):
try:
upload_request = UploadFileRequest(self._bucket,
unicode(cos_path),
local_path,
insert_only=insert_only)
upload_file_ret = self._cos_api.upload_file(upload_request)
if upload_file_ret[u'code'] != 0:
logger.warn("upload failed:" + str(upload_file_ret))
raise OSError("UploadError: " + str(upload_file_ret))
else:
break
except:
pass
else:
raise IOError("upload failed")
def list(self):
if self._prefix_dir is None:
for i in self.dfs('/'):
yield i
else:
for i in self.dfs(self._prefix_dir):
yield i
def dfs(self, path):
print "DFS: {path}".format(path=to_utf8(path))
_finish = False
_context = u''
max_retry = self._max_retry
path = to_unicode(path)
while not _finish:
request = ListFolderRequest(bucket_name=self._bucket,
cos_path=path,
context=_context)
ret = self._cos_api.list_folder(request)
if ret['code'] != 0:
max_retry -= 1
else:
_finish = ret['data']['listover']
_context = ret['data']['context']
for item in ret['data']['infos']:
if 'filelen' in item:
try:
key = "{prefix}{filename}".format(
prefix=path, filename=item['name'])
yield Task(key, item['filelen'], None)
except:
pass
else:
_sub_dir = "{prefix}{filename}".format(
prefix=path.encode('utf-8'),
filename=item['name'].encode('utf-8'))
for i in self.dfs(_sub_dir):
yield i
if max_retry == 0:
_finish = True
def exists(self, task):
_path = task.key
_size = task.size
if not _path.startswith('/'):
_path = '/' + _path
logger = getLogger(__name__)
# logger.info("func: exists: " + str(_path))
if self._prefix_dir:
_path = self._prefix_dir + _path
if isinstance(_path, str):
_path = _path.decode('utf-8')
request = StatFileRequest(self._bucket, _path)
ret = self._cos_api.stat_file(request)
logger.debug("ret: " + str(ret))
# import json
# v = json.loads(ret)
if ret['code'] != 0:
# logger.warn("error code: " + str(ret['code']))
return False
if ret['data']['filelen'] != ret['data']['filesize']:
logger.warn(
"file is broken, filelen: {len}, filesize: {size}".format(
len=ret['data']['filelen'], size=ret['data']['filesize']))
return False
elif _size is not None and ret['data']['filelen'] != _size:
return False
elif ret['data']['filelen'] != _size:
return False
return True
def cos_demo():
# 设置用户属性, 包括appid, secret_id和secret_key
# 这些属性可以在cos控制台获取(https://console.qcloud.com/cos)
appid = 111 # 替换为用户的appid
secret_id = u'' # 替换为用户的secret_id
secret_key = u'' # 替换为用户的secret_key
region = "shanghai" # # 替换为用户的region，目前可以为 shanghai/guangzhou
cos_client = CosClient(appid, secret_id, secret_key, region)
# 设置要操作的bucket
bucket = u''
############################################################################
# 文件操作 #
############################################################################
# 1. 上传文件(默认不覆盖)
# 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt
# 默认不覆盖, 如果cos上文件存在,则会返回错误
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_1.txt')
upload_file_ret = cos_client.upload_file(request)
logger.info("upload file, return message: " + str(upload_file_ret))
# 分片上传大文件
request = UploadSliceFileRequest(bucket, u'/sample_bigfile.txt',
u'local_bigfile.txt')
ret = cos_client.upload_slice_file(request)
logger.info("slice upload, return message: " + str(ret))
# 2. 上传文件(覆盖文件)
# 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt
with open('local_file_2.txt', 'w') as f:
f.write("hello world2")
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_2.txt')
request.set_insert_only(0) # 设置允许覆盖
upload_file_ret = cos_client.upload_file(request)
logger.info('overwrite file, return message:' + str(upload_file_ret))
# 3. 获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
logger.info('stat file, return message: ' + str(stat_file_ret))
# 4. 更新文件属性
request = UpdateFileRequest(bucket, u'/sample_file.txt')
request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性
request.set_authority(u'eWRPrivate') # 设置文件的权限
request.set_cache_control(u'cache_xxx') # 设置Cache-Control
request.set_content_type(u'application/text') # 设置Content-Type
request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition
request.set_content_language(u'english') # 设置Content-Language
request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性
request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性
update_file_ret = cos_client.update_file(request)
logger.info('update file, return message: ' + str(update_file_ret))
# 5. 更新后再次获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
logger.info('stat file, return message: ' + str(stat_file_ret))
############################################################################
# 目录操作 #
############################################################################
# 1. 生成目录, 目录名为sample_folder
request = CreateFolderRequest(bucket, u'/sample_folder/')
create_folder_ret = cos_client.create_folder(request)
logger.info('create folder ret:' + str(create_folder_ret))
# 2. 更新目录的biz_attr属性
request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录')
update_folder_ret = cos_client.update_folder(request)
logger.info('update folder ret:' + str(update_folder_ret))
# 3. 获取目录属性
request = StatFolderRequest(bucket, u'/sample_folder/')
stat_folder_ret = cos_client.stat_folder(request)
logger.info("stat folder, return message: " + str(stat_folder_ret))
# 4. list目录, 获取目录下的成员
request = ListFolderRequest(bucket, u'/sample_folder/')
list_folder_ret = cos_client.list_folder(request)
logger.info("list folder, return message: " + str(list_folder_ret))
# 5. 删除目录
request = DelFolderRequest(bucket, u'/sample_folder/')
delete_folder_ret = cos_client.del_folder(request)
logger.info("delete folder, return message: " + str(delete_folder_ret))
# coding=utf-8
from qcloud_cos import CosClient
from qcloud_cos import UploadFileRequest
app_id = 10056151
# secret id
# AKIDtNtth4SdBNy5aidDI9wHEiPDYXvRsfbV
secret_id = u'AKIDtNtth4SdBNy5aidDI9wHEiPDYXvRsfbV'
# secret key
# 7pdJ5w9Uq0OGderacgWaA4UCerip5AVe
secret_key = u'7pdJ5w9Uq0OGderacgWaA4UCerip5AVe'
# cos client
cos_client = CosClient(app_id, secret_id, secret_key)
# set bucket
bucket = u'portal'
# upload
request = UploadFileRequest(bucket, u'/card_image/test_upload.jpg',
u'./suit.jpg')
ret = cos_client.upload_file(request)
print "upload file ret:", repr(ret)
def cos_demo():
# 设置用户属性, 包括appid, secret_id和secret_key
# 这些属性可以在cos控制台获取(https://console.qcloud.com/cos)
appid = 111 # 替换为用户的appid
secret_id = u'' # 替换为用户的secret_id
secret_key = u'' # 替换为用户的secret_key
region = "shanghai" # # 替换为用户的region，目前可以为 shanghai/guangzhou
cos_client = CosClient(appid, secret_id, secret_key, region)
# 设置要操作的bucket
bucket = u''
############################################################################
# 文件操作 #
############################################################################
# 1. 上传文件(默认不覆盖)
# 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt
# 默认不覆盖, 如果cos上文件存在,则会返回错误
request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_1.txt')
upload_file_ret = cos_client.upload_file(request)
logger.info("upload file, return message: " + str(upload_file_ret))
# 分片上传大文件
request = UploadSliceFileRequest(bucket, u'/sample_bigfile.txt', u'local_bigfile.txt')
ret = cos_client.upload_slice_file(request)
logger.info("slice upload, return message: " + str(ret))
# 2. 上传文件(覆盖文件)
# 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt
with open('local_file_2.txt', 'w') as f:
f.write("hello world2")
request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_2.txt')
request.set_insert_only(0) # 设置允许覆盖
upload_file_ret = cos_client.upload_file(request)
logger.info('overwrite file, return message:' + str(upload_file_ret))
# 3. 获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
logger.info('stat file, return message: ' + str(stat_file_ret))
# 4. 更新文件属性
request = UpdateFileRequest(bucket, u'/sample_file.txt')
request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性
request.set_authority(u'eWRPrivate') # 设置文件的权限
request.set_cache_control(u'cache_xxx') # 设置Cache-Control
request.set_content_type(u'application/text') # 设置Content-Type
request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition
request.set_content_language(u'english') # 设置Content-Language
request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性
request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性
update_file_ret = cos_client.update_file(request)
logger.info('update file, return message: ' + str(update_file_ret))
# 5. 更新后再次获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
logger.info('stat file, return message: ' + str(stat_file_ret))
############################################################################
# 目录操作 #
############################################################################
# 1. 生成目录, 目录名为sample_folder
request = CreateFolderRequest(bucket, u'/sample_folder/')
create_folder_ret = cos_client.create_folder(request)
logger.info('create folder ret:' + str(create_folder_ret))
# 2. 更新目录的biz_attr属性
request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录')
update_folder_ret = cos_client.update_folder(request)
logger.info('update folder ret:' + str(update_folder_ret))
# 3. 获取目录属性
request = StatFolderRequest(bucket, u'/sample_folder/')
stat_folder_ret = cos_client.stat_folder(request)
logger.info("stat folder, return message: " + str(stat_folder_ret))
# 4. list目录, 获取目录下的成员
request = ListFolderRequest(bucket, u'/sample_folder/')
list_folder_ret = cos_client.list_folder(request)
logger.info("list folder, return message: " + str(list_folder_ret))
# 5. 删除目录
request = DelFolderRequest(bucket, u'/sample_folder/')
delete_folder_ret = cos_client.del_folder(request)
logger.info("delete folder, return message: " + str(delete_folder_ret))
#coding=utf-8
import json
from qcloud_cos import CosClient
from qcloud_cos import UploadFileRequest, DelFileRequest,CreateFolderRequest,ListFolderRequest,StatFolderRequest,StatFileRequest,DelFolderRequest
from qcloud_image import Client
from qcloud_image import CIUrl, CIFile, CIBuffer, CIUrls, CIFiles, CIBuffers
appid = 100000000 # 替换为用户的appid
secret_id = u'' # 替换为用户的secret_id
secret_key = u'' # 替换为用户的secret_key
region_info = u"sh" # 替换为用户的region，例如 sh 表示华东园区, gz 表示华南园区, tj 表示华北园区
bucket = u'test'
cos_client = CosClient(appid, secret_id, secret_key, region=region_info)
#查看文件属性
def stat_file_ret():
request = StatFileRequest(bucket, u'/meizitu/meizitu/0067Dm70gy1fj4j5dexhbj30u011hwjl.jpg')
stat_file_ret = cos_client.stat_file(request)
print stat_file_ret
#上传文件
def upload_file(filename):
request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_1.txt')
upload_file_ret = cos_client.upload_file(request)
#删除文件
def del_file(filename):
request = DelFileRequest(bucket, u'%s'%filename)
del_ret = cos_client.del_file(request)
return del_ret
#删除目录
def cos_demo():
# 设置用户属性, 包括appid, secret_id和secret_key
# 这些属性可以在cos控制台获取(https://console.qcloud.com/cos)
appid = 100000 # 替换为用户的appid
secret_id = u'xxxxxxxx' # 替换为用户的secret_id
secret_key = u'xxxxxxx' # 替换为用户的secret_key
cos_client = CosClient(appid, secret_id, secret_key)
# 设置要操作的bucket
bucket = u'mybucket'
############################################################################
# 文件操作 #
############################################################################
# 1. 上传文件(默认不覆盖)
# 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt
# 默认不覆盖, 如果cos上文件存在,则会返回错误
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_1.txt')
upload_file_ret = cos_client.upload_file(request)
print 'upload file ret:', repr(upload_file_ret)
# 2. 上传文件(覆盖文件)
# 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt
request = UploadFileRequest(bucket, u'/sample_file.txt',
u'local_file_2.txt')
request.set_insert_only(0) # 设置允许覆盖
upload_file_ret = cos_client.upload_file(request)
print 'overwrite file ret:', repr(upload_file_ret)
# 3. 获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
print 'stat file ret:', repr(stat_file_ret)
# 4. 更新文件属性
request = UpdateFileRequest(bucket, u'/sample_file.txt')
request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性
request.set_authority(u'eWRPrivate') # 设置文件的权限
request.set_cache_control(u'cache_xxx') # 设置Cache-Control
request.set_content_type(u'application/text') # 设置Content-Type
request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition
request.set_content_language(u'english') # 设置Content-Language
request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性
request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性
update_file_ret = cos_client.update_file(request)
print 'update file ret:', repr(update_file_ret)
# 5. 更新后再次获取文件属性
request = StatFileRequest(bucket, u'/sample_file.txt')
stat_file_ret = cos_client.stat_file(request)
print 'stat file ret:', repr(stat_file_ret)
# 6. 移动文件, 将sample_file.txt移动位sample_file_move.txt
request = MoveFileRequest(bucket, u'/sample_file.txt',
u'/sample_file_move.txt')
stat_file_ret = cos_client.move_file(request)
print 'move file ret:', repr(stat_file_ret)
# 7. 删除文件
request = DelFileRequest(bucket, u'/sample_file_move.txt')
del_ret = cos_client.del_file(request)
print 'del file ret:', repr(del_ret)
############################################################################
# 目录操作 #
############################################################################
# 1. 生成目录, 目录名为sample_folder
request = CreateFolderRequest(bucket, u'/sample_folder/')
create_folder_ret = cos_client.create_folder(request)
print 'create folder ret:', create_folder_ret
# 2. 更新目录的biz_attr属性
request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录')
update_folder_ret = cos_client.update_folder(request)
print 'update folder ret:', repr(update_folder_ret)
# 3. 获取目录属性
request = StatFolderRequest(bucket, u'/sample_folder/')
stat_folder_ret = cos_client.stat_folder(request)
print 'stat folder ret:', repr(stat_folder_ret)
# 4. list目录, 获取目录下的成员
request = ListFolderRequest(bucket, u'/sample_folder/')
list_folder_ret = cos_client.list_folder(request)
print 'list folder ret:', repr(list_folder_ret)
# 5. 删除目录
request = DelFolderRequest(bucket, u'/sample_folder/')
delete_folder_ret = cos_client.del_folder(request)
print 'delete folder ret:', repr(delete_folder_ret)
def sync(self):
# 本地文件目录列表
local_file_dir_info = LocalFileDirInfo(self.config.get_local_path())
# 记录的要删除的文件和目录列表
del_file_dict = self.db.file_stat_dict
del_dir_dict = self.db.dir_stat_dict
# 记录的要创建的文件和目录列表
create_file_dict = {}
create_dir_dict = {}
# 文件同步状态校验
for local_file_stat in local_file_dir_info.file_stat_list:
local_file_path = local_file_stat.file_path
# 如果数据库中记录cos中也存在, 则判断下文件是否发生了改变
if (self.db.file_stat_dict.has_key(local_file_path)):
# 如果文件内容未发生变化, 则不用同步, 否则进行同步
if (FileStat.compare(
local_file_stat,
self.db.file_stat_dict[local_file_path]) == 0):
del self.db.file_stat_dict[local_file_path]
else:
create_file_dict[local_file_path] = local_file_stat
# 如果数据库中不存在相关记录, 则插入到要创建的文件列表
else:
create_file_dict[local_file_path] = local_file_stat
# 目录同步状态校验
for local_dir_stat in local_file_dir_info.dir_stat_list:
local_dir_path = local_dir_stat.file_path
# 如果数据库中记录cos中也存在, 则判断下目录是否发生了改变
if (self.db.dir_stat_dict.has_key(local_dir_path)):
del self.db.dir_stat_dict[local_dir_path]
# 如果数据库中不存在相关记录, 则插入到要创建的文件列表
else:
create_dir_dict[local_dir_path] = local_dir_stat
# 生成cos客户端
appid = int(self.config.get_appid())
secret_id = self.config.get_secret_id()
secret_key = self.config.get_secret_key()
bucket = self.config.get_bucket()
timeout = self.config.get_timeout()
cos_client = CosClient(appid, secret_id, secret_key)
cos_config = CosConfig()
cos_config.set_timeout(timeout)
if self.config.get_enable_https() == 1:
cos_config.enable_https()
cos_client.set_config(cos_config)
# 创建要同步的cos 目录
self._create_cos_target_dir(cos_client, bucket)
# 进行同步, 同步顺序如下
# 1 删除文件(之前本地有，现在没有的文件, 或者发生修改的文件)
self._del_cos_sync_file(cos_client, bucket, del_file_dict)
# 2 删除目录(之前本地有，现在没有的目录)
self._del_cos_sync_dir(cos_client, bucket, del_dir_dict)
# 3 创建目录(新建的目录)
self._create_cos_sync_dir(cos_client, bucket, create_dir_dict)
# 4 创建文件(新建的文件)
self._create_cos_sync_file(cos_client, bucket, create_file_dict)
