def upload_cos(file): """ CVM 上报 https 错误:取消 https 试试 """ appid = 100000 # 根据自己的情况填写 secret_id = u'# 根据自己的情况填写' secret_key = u'# 根据自己的情况填写' region = '# 根据自己的情况填写' bucket = u'# 根据自己的情况填写' cos_client = CosClient(appid, secret_id, secret_key, region) cos_path = u'/' + file request = UploadFileRequest(bucket, cos_path, unicode(file)) request.set_insert_only(0) start = time.time() resp = cos_client.upload_file(request) # assert resp['code'] == 0 if resp['code'] != 0: return 0 end = time.time() elapsed = (end - start) * 1000.0 print u'本次 COS 文件上传时间为 %0.3f ms' % elapsed return elapsed
def upload(self, path): """ cos_path:/news/jiemian/image/ :param path :return: """ counter = 0 url = '' while counter != 10: try: # 得到hash uploadName = path.replace('image\\', '') request = UploadFileRequest(u"crawler", self.cos_path + uploadName, self.local_path + path, insert_only=0) upload_file_ret = self.cos_client.upload_file(request) if upload_file_ret['code'] == 0: data = upload_file_ret['data'] or {} url = data['source_url'] print u'上传成功 ' + url else: print u'上传图片失败', upload_file_ret break except Exception as e: counter += 1 TimerUtil.sleep(10) return url
def upload(self, task, local_path): cos_path = task.key if not cos_path.startswith('/'): cos_path = '/' + cos_path if self._prefix_dir: cos_path = self._prefix_dir + cos_path if isinstance(local_path, unicode): local_path.encode('utf-8') insert_only = 0 if self._overwrite else 1 for i in range(10): logger.info("upload file: %s, with retry %d, need size: %d", task.key, i, task.size) try: upload_request = UploadFileRequest(self._bucket, unicode(cos_path), local_path, insert_only=insert_only) upload_file_ret = self._cos_api.upload_file(upload_request) if upload_file_ret[u'code'] != 0: logger.warn("error: cos path: %s, upload failed: %s", cos_path, str(upload_file_ret)) raise OSError("UploadError: " + str(upload_file_ret)) else: logger.info( "upload task: %s, to cos path: %s, success, break", task.key, cos_path) break except Exception as e: logger.error("try: %d, error: %s", i, str(e)) pass else: raise IOError("upload failed")
def upload_file(cos_client, bucket, cos_path, db, file_stat): global cos_task_statics local_path = file_stat.file_path if not isinstance(local_path, unicode): local_path = local_path.decode(fs_encoding) retry_index = 0 retry_max_cnt = 3 while retry_index < retry_max_cnt: request = UploadFileRequest(bucket, cos_path, local_path) request.set_insert_only(0) op_ret = cos_client.upload_file(request) if cos_op_success(op_ret): db.update_record(file_stat) cos_task_statics.add_upload_file_ok() print_op_ok("uploadFile", op_ret, local_path) return True if is_upload_server_error(op_ret): del_request = DelFileRequest(bucket, cos_path) cos_client.del_file(del_request) retry_index += 1 if retry_index == retry_max_cnt: cos_task_statics.add_upload_file_fail() print_op_fail("uploadFile", op_ret, local_path) return False
def update_packages(url_list): cos_client = CosClient(app_id, secret_id, secret_key, region) for package in url_list: print package.filename if package.status == 'need update': print 'update' elif package.status == 'not exist': print 'downloading', package.filename urllib.urlretrieve(package.url, os.path.join('packages', 'CTS', package.filename), reporthook=report) sys.stdout.flush() else: print 'skip', package.filename request = StatFileRequest(bucket, unicode('/cts/' + package.filename)) stat_file_ret = cos_client.stat_file(request) if stat_file_ret['code'] != 0: file_path = os.path.join(os.path.dirname(__file__), 'packages', 'CTS', package.filename) request = UploadFileRequest(bucket, unicode('/cts/' + package.filename), unicode(file_path)) upload_file_ret = cos_client.upload_file(request) print upload_file_ret
def upload(self, task, local_path): cos_path = task.key if not cos_path.startswith('/'): cos_path = '/' + cos_path if self._prefix_dir: cos_path = self._prefix_dir + cos_path if isinstance(local_path, unicode): local_path.encode('utf-8') insert_only = 0 if self._overwrite else 1 for i in range(10): try: upload_request = UploadFileRequest(self._bucket, unicode(cos_path), local_path, insert_only=insert_only) upload_file_ret = self._cos_api.upload_file(upload_request) if upload_file_ret[u'code'] != 0: logger.warn("upload failed:" + str(upload_file_ret)) raise OSError("UploadError: " + str(upload_file_ret)) else: break except: pass else: raise IOError("upload failed")
def upload_file(cos_client, bucket, key, local_file_path): request = UploadFileRequest(bucket.decode('utf-8'), key.decode('utf-8'), local_file_path.decode('utf-8')) upload_file_ret = cos_client.upload_file(request) if upload_file_ret['code'] == 0: logger.info("Upload data map file [%s] Success" % key) return 0 else: logger.error("Upload data map file [%s] Failed, err: %s" % (key, upload_file_ret['message'])) return -1
def main_handler(event, context): logger.info("start main handler") for record in event['Records']: try: bucket = record['cos']['cosBucket']['name'] key = record['cos']['cosObject']['key'] key = key.replace('/' + str(appid) + '/' + bucket, '') download_path = '/tmp/{}{}'.format(uuid.uuid4(), key.split('/')[-1]) upload_name = '{}{}'.format( file_name_p.findall(key.split('/')[-1])[0], 'png') upload_path = '/tmp/rounded-{}'.format(upload_name) print("Key is " + key) # file name in bucket print("Get from [%s] to download file [%s]" % (bucket, key)) # download image from cos request = DownloadFileRequest(bucket, key, download_path) download_file_ret = cos_client.download_file(request) if download_file_ret['code'] == 0: # TODO change rad as Incoming parameter rad = 100 logger.info("Download file [%s] Success" % key) logger.info("Image compress function start") starttime = datetime.datetime.now() #compress image here round_image(download_path, upload_path, rad) endtime = datetime.datetime.now() logger.info("compress image take " + str((endtime - starttime).microseconds / 1000) + "ms") #upload the compressed image to resized bucket u_key = '{}{}'.format(file_name_p.findall(key)[0], 'png') request = UploadFileRequest(u'%sresized' % bucket, u_key.decode('utf-8'), upload_path.decode('utf-8')) upload_file_ret = cos_client.upload_file(request) logger.info("upload image, return message: " + str(upload_file_ret)) #delete local file delete_local_file(str(download_path)) delete_local_file(str(upload_path)) else: logger.error("Download file [%s] Failed, err: %s" % (key, download_file_ret['message'])) return -1 except Exception as e: print(e) print( 'Error getting object {} from bucket {}. Make sure the object exists and your bucket is in the same region as this function.' .format(key, bucket)) raise e
def cos_demo(mkdir, file_path): appid = xxxxxx # 替换为用户的appid secret_id = u'xxxxxxx' # 替换为用户的secret_id secret_key = u'xxxxxxx' # 替换为用户的secret_key region = "tj" # # 替换为用户的region,目前可以为 shanghai/guangzhou cos_client = CosClient(appid, secret_id, secret_key, region) bucket = u'xxxx' request = UploadFileRequest(bucket, u'/' + mkdir + '/' + file_path + '', u'' + file_path + '') upload_file_ret = cos_client.upload_file(request) print logger.info("upload file, return message: " + str(upload_file_ret))
def upload(self, cos_path, local_path): if not cos_path.startswith('/'): cos_path = '/' + cos_path if self._prefix_dir: cos_path = self._prefix_dir + cos_path if isinstance(local_path, unicode): local_path.encode('utf-8') upload_request = UploadFileRequest(self._bucket, unicode(cos_path), local_path) upload_file_ret = self._cos_api.upload_file(upload_request) if upload_file_ret[u'code'] != 0: raise OSError("UploadError: " + str(upload_file_ret))
def generate_html(packages): page = markup.page() page.init(title="CTS Packages", ) page.pre() for package in packages: page.a(package.filename, href=cos_info['cos_url'] + package.filename) page.pre.close() with open('index.html', 'w') as t: t.write(str(page)) t.close() cos_client = CosClient(app_id, secret_id, secret_key, region) request = UploadFileRequest(bucket, unicode('/cts/index.html'), unicode('index.html')) upload_file_ret = cos_client.upload_file(request) print upload_file_ret
def uploadToCos(cos_client, bucket, root, localFile): # try 10 times, if fail then print error. succes = times = 0 request = UploadFileRequest(bucket, localFile, root + localFile) request.set_insert_only(0) # 0是允许覆盖 1是不允许 while succes == 0 and times < 10: times += 1 try: upload_file_ret = cos_client.upload_file(request) if upload_file_ret['message'] == u'SUCCESS': succes = 1 print 'Upload | %-10s | %s' % (upload_file_ret['message'], localFile) except: pass if times == 10: print '===Error===: %s upload failed' % (localFile)
def upload_file(cos_client, bucket, cos_path, db, file_stat): global cos_task_statics local_path = file_stat.file_path request = UploadFileRequest(bucket, cos_path, local_path) request.set_insert_only(0) # 如果cos上有路径则覆盖文件 op_ret = cos_client.upload_file(request) if cos_op_success(op_ret): db.update_record(file_stat) cos_task_statics.update_statics("upload_file_success") print_op_ok("uploadFile", op_ret, local_path) return True else: # cos_task_statics.update_statics("upload_file_fail") # print_op_fail("uploadFile", op_ret, local_path) # return False cos_task_statics.update_statics("upload_file_retries") print_op_fail("uploadFile (to retry)", op_ret, local_path) delete_file(cos_client, bucket, cos_path, db, file_stat) time.sleep(1) return upload_file(cos_client, bucket, cos_path, db, file_stat)
def uploadfile(): ftp = ftpconnect() myCosClient = GetCosClient() datapath = "/pub/data/noaa/isd-lite/" beginyear = 2017 endyear = 2017 year = beginyear while year <= endyear: ftppath = datapath + str(year) filelist = ftp.nlst(ftppath) cospath = u'/' + str(year) + u'/' ret = myCosClient.create_folder( CreateFolderRequest(u'fredcos', cospath)) print ret for file in filelist: localfilepaths = file.split("/") localfilepath = localfilepaths[len(localfilepaths) - 1] bufsize = 1024 localfile = open(localfilepath, 'wb') ftp.retrbinary('RETR ' + file, localfile.write, bufsize) cosfilepath = cospath + localfilepath request = UploadFileRequest(u'fredtest', cosfilepath, localfilepath.decode("utf-8")) localfilepath.decode() upload_file_ret = myCosClient.upload_file(request) print upload_file_ret localfile.close() if os.path.exists(localfilepath): os.remove(localfilepath) year = year + 1 ftp.close()
def upload(file_bucket, filename_in_cos, file_local_path, cos_client): """upload file to cos bucket and it will replace the file with the same name. Args: file_bucket (str): bucket name. filename_in_cos (str): file name in the bucket, which should be started with "/". file_local_path (str): local path in the unix path format, for example "/tmp/xxx.jpg". cos_client (object): cos client created by Tencent COS SDK """ request = UploadFileRequest(file_bucket, filename_in_cos, file_local_path, insert_only=0) upload_file_ret = cos_client.upload_file(request) logger.info("Upload image, return message: " + str(upload_file_ret)) if upload_file_ret['code'] != 0: logger.info("Fail Upload file [%s]" % filename_in_cos) return -1 else: logger.info("SUC Upload file [%s]" % filename_in_cos) return 0
def upload(self, path): """ cos_path:/news/wangyi/image/ :param path :return: """ # 得到hash uploadName = path.replace('full/', '') request = UploadFileRequest(u"crawler", self.cos_path + uploadName, self.local_path + path, insert_only=0) upload_file_ret = self.cos_client.upload_file(request) url = '' if upload_file_ret['code'] == 0: data = upload_file_ret['data'] or {} url = data['source_url'] print u'上传成功', url else: print u'上传图片失败', upload_file_ret return url
def uploadList(self, listFile): """ cos_path:/news/jiemian/image/ :param listFile: [{url,path},{}] :return: """ results = [] for item in listFile: # 得到hash path = item['path'] uploadName = path.replace(u'full/', '') request = UploadFileRequest(u"crawler", self.cos_path + uploadName, self.local_path + path, insert_only=0) upload_file_ret = self.cos_client.upload_file(request) if upload_file_ret['code'] == 0: data = upload_file_ret['data'] or {} url = data['source_url'] results.append(url) print u'upload successfully', url else: print u'fail to uplaod ', upload_file_ret return results
# coding=utf-8 from qcloud_cos import CosClient from qcloud_cos import UploadFileRequest app_id = 10056151 # secret id # AKIDtNtth4SdBNy5aidDI9wHEiPDYXvRsfbV secret_id = u'AKIDtNtth4SdBNy5aidDI9wHEiPDYXvRsfbV' # secret key # 7pdJ5w9Uq0OGderacgWaA4UCerip5AVe secret_key = u'7pdJ5w9Uq0OGderacgWaA4UCerip5AVe' # cos client cos_client = CosClient(app_id, secret_id, secret_key) # set bucket bucket = u'portal' # upload request = UploadFileRequest(bucket, u'/card_image/test_upload.jpg', u'./suit.jpg') ret = cos_client.upload_file(request) print "upload file ret:", repr(ret)
from qcloud_cos import UploadFileRequest from qcloud_cos import CosClient import sys if __name__ == '__main__': if len(sys.argv) < 3: print "argv error" bucket = u'koprvhdix117' app_id = 10038234 secret_id = u'AKIDkTQGMuCeJvtTTlqg911BfF393ghumqHp' secret_key = u'ZE1uBa6jfbsB0vVyfbWhw5JuZKPwaEwh' cos_client = CosClient(app_id, secret_id, secret_key) if sys.argv[1] == "upload": cos_path = unicode(sys.argv[2]) local_path = unicode(sys.argv[3]) request = UploadFileRequest(bucket, cos_path, local_path) upload_file_ret = cos_client.upload_file(request) print upload_file_ret elif sys.argv[1] == "delete": cos_path = unicode(sys.argv[2]) request = DelFileRequest(bucket, cos_path) delete_file_ret = cos_client.del_file(request) print delete_file_ret else: print "argv error"
#!/usr/bin/python # -*- coding: UTF-8 -*- from qcloud_cos import CosClient from qcloud_cos import UploadFileRequest from qcloud_cos import UploadSliceFileRequest from qcloud_cos import UpdateFileRequest from qcloud_cos import UpdateFolderRequest from qcloud_cos import DelFileRequest from qcloud_cos import DelFolderRequest from qcloud_cos import CreateFolderRequest from qcloud_cos import StatFileRequest from qcloud_cos import StatFolderRequest from qcloud_cos import ListFolderRequest import logging import sys appid = 1253506722 secret_id = u'AKIDjKjx2vmhePxKldwwKaeVEl5C8vX8pAPS' secret_key = u'0rVxEKDuOr9JH5h8OnOn026zhT8bswpi' region_info = "tj" cos_client = CosClient(appid, secret_id, secret_key, region=region_info) request = UploadFileRequest(u"codingdie01", sys.argv[2].decode("utf-8"), sys.argv[1].decode("utf-8")) result = cos_client.upload_file(request) print result
def upload_file(filename): request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_1.txt') upload_file_ret = cos_client.upload_file(request)
def cos_demo(): # 设置用户属性, 包括appid, secret_id和secret_key # 这些属性可以在cos控制台获取(https://console.qcloud.com/cos) appid = 111 # 替换为用户的appid secret_id = u'' # 替换为用户的secret_id secret_key = u'' # 替换为用户的secret_key region = "shanghai" # # 替换为用户的region,目前可以为 shanghai/guangzhou cos_client = CosClient(appid, secret_id, secret_key, region) # 设置要操作的bucket bucket = u'' ############################################################################ # 文件操作 # ############################################################################ # 1. 上传文件(默认不覆盖) # 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt # 默认不覆盖, 如果cos上文件存在,则会返回错误 request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_1.txt') upload_file_ret = cos_client.upload_file(request) logger.info("upload file, return message: " + str(upload_file_ret)) # 分片上传大文件 request = UploadSliceFileRequest(bucket, u'/sample_bigfile.txt', u'local_bigfile.txt') ret = cos_client.upload_slice_file(request) logger.info("slice upload, return message: " + str(ret)) # 2. 上传文件(覆盖文件) # 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt with open('local_file_2.txt', 'w') as f: f.write("hello world2") request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_2.txt') request.set_insert_only(0) # 设置允许覆盖 upload_file_ret = cos_client.upload_file(request) logger.info('overwrite file, return message:' + str(upload_file_ret)) # 3. 获取文件属性 request = StatFileRequest(bucket, u'/sample_file.txt') stat_file_ret = cos_client.stat_file(request) logger.info('stat file, return message: ' + str(stat_file_ret)) # 4. 更新文件属性 request = UpdateFileRequest(bucket, u'/sample_file.txt') request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性 request.set_authority(u'eWRPrivate') # 设置文件的权限 request.set_cache_control(u'cache_xxx') # 设置Cache-Control request.set_content_type(u'application/text') # 设置Content-Type request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition request.set_content_language(u'english') # 设置Content-Language request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性 request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性 update_file_ret = cos_client.update_file(request) logger.info('update file, return message: ' + str(update_file_ret)) # 5. 更新后再次获取文件属性 request = StatFileRequest(bucket, u'/sample_file.txt') stat_file_ret = cos_client.stat_file(request) logger.info('stat file, return message: ' + str(stat_file_ret)) ############################################################################ # 目录操作 # ############################################################################ # 1. 生成目录, 目录名为sample_folder request = CreateFolderRequest(bucket, u'/sample_folder/') create_folder_ret = cos_client.create_folder(request) logger.info('create folder ret:' + str(create_folder_ret)) # 2. 更新目录的biz_attr属性 request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录') update_folder_ret = cos_client.update_folder(request) logger.info('update folder ret:' + str(update_folder_ret)) # 3. 获取目录属性 request = StatFolderRequest(bucket, u'/sample_folder/') stat_folder_ret = cos_client.stat_folder(request) logger.info("stat folder, return message: " + str(stat_folder_ret)) # 4. list目录, 获取目录下的成员 request = ListFolderRequest(bucket, u'/sample_folder/') list_folder_ret = cos_client.list_folder(request) logger.info("list folder, return message: " + str(list_folder_ret)) # 5. 删除目录 request = DelFolderRequest(bucket, u'/sample_folder/') delete_folder_ret = cos_client.del_folder(request) logger.info("delete folder, return message: " + str(delete_folder_ret))
#!/usr/bin/env python # coding=utf-8 from qcloud_cos import CosClient from qcloud_cos import UploadFileRequest appid = 100000 # 替换为你的appid secret_id = u'' # 替换为你的secret_id secret_key = u'' # 替换为你的secret_key bucket = u'' # 替换为你要操作的bucket名 region = "tj" # 替换为该bucket所属的地区代码, tj/sh/gz/sgp cos_client = CosClient(appid, secret_id, secret_key, region) ################################## # 文件操作 # ################################## # 上传文件(默认不允许覆盖) # 将本地的 upload_example.txt 上传到 bucket 的根分区下,并命名为 upload_sample_python.txt request = UploadFileRequest(bucket, u'/upload_sample_python.txt', u'/data/upload/upload_example.txt') upload_file_response = cos_client.upload_file(request) print upload_file_response
def cos_demo(): # 设置用户属性, 包括appid, secret_id和secret_key # 这些属性可以在cos控制台获取(https://console.qcloud.com/cos) appid = 100000 # 替换为用户的appid secret_id = u'xxxxxxxx' # 替换为用户的secret_id secret_key = u'xxxxxxx' # 替换为用户的secret_key cos_client = CosClient(appid, secret_id, secret_key) # 设置要操作的bucket bucket = u'mybucket' ############################################################################ # 文件操作 # ############################################################################ # 1. 上传文件(默认不覆盖) # 将本地的local_file_1.txt上传到bucket的根分区下,并命名为sample_file.txt # 默认不覆盖, 如果cos上文件存在,则会返回错误 request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_1.txt') upload_file_ret = cos_client.upload_file(request) print 'upload file ret:', repr(upload_file_ret) # 2. 上传文件(覆盖文件) # 将本地的local_file_2.txt上传到bucket的根分区下,覆盖已上传的sample_file.txt request = UploadFileRequest(bucket, u'/sample_file.txt', u'local_file_2.txt') request.set_insert_only(0) # 设置允许覆盖 upload_file_ret = cos_client.upload_file(request) print 'overwrite file ret:', repr(upload_file_ret) # 3. 获取文件属性 request = StatFileRequest(bucket, u'/sample_file.txt') stat_file_ret = cos_client.stat_file(request) print 'stat file ret:', repr(stat_file_ret) # 4. 更新文件属性 request = UpdateFileRequest(bucket, u'/sample_file.txt') request.set_biz_attr(u'这是个demo文件') # 设置文件biz_attr属性 request.set_authority(u'eWRPrivate') # 设置文件的权限 request.set_cache_control(u'cache_xxx') # 设置Cache-Control request.set_content_type(u'application/text') # 设置Content-Type request.set_content_disposition(u'ccccxxx.txt') # 设置Content-Disposition request.set_content_language(u'english') # 设置Content-Language request.set_x_cos_meta(u'x-cos-meta-xxx', u'xxx') # 设置自定义的x-cos-meta-属性 request.set_x_cos_meta(u'x-cos-meta-yyy', u'yyy') # 设置自定义的x-cos-meta-属性 update_file_ret = cos_client.update_file(request) print 'update file ret:', repr(update_file_ret) # 5. 更新后再次获取文件属性 request = StatFileRequest(bucket, u'/sample_file.txt') stat_file_ret = cos_client.stat_file(request) print 'stat file ret:', repr(stat_file_ret) # 6. 移动文件, 将sample_file.txt移动位sample_file_move.txt request = MoveFileRequest(bucket, u'/sample_file.txt', u'/sample_file_move.txt') stat_file_ret = cos_client.move_file(request) print 'move file ret:', repr(stat_file_ret) # 7. 删除文件 request = DelFileRequest(bucket, u'/sample_file_move.txt') del_ret = cos_client.del_file(request) print 'del file ret:', repr(del_ret) ############################################################################ # 目录操作 # ############################################################################ # 1. 生成目录, 目录名为sample_folder request = CreateFolderRequest(bucket, u'/sample_folder/') create_folder_ret = cos_client.create_folder(request) print 'create folder ret:', create_folder_ret # 2. 更新目录的biz_attr属性 request = UpdateFolderRequest(bucket, u'/sample_folder/', u'这是一个测试目录') update_folder_ret = cos_client.update_folder(request) print 'update folder ret:', repr(update_folder_ret) # 3. 获取目录属性 request = StatFolderRequest(bucket, u'/sample_folder/') stat_folder_ret = cos_client.stat_folder(request) print 'stat folder ret:', repr(stat_folder_ret) # 4. list目录, 获取目录下的成员 request = ListFolderRequest(bucket, u'/sample_folder/') list_folder_ret = cos_client.list_folder(request) print 'list folder ret:', repr(list_folder_ret) # 5. 删除目录 request = DelFolderRequest(bucket, u'/sample_folder/') delete_folder_ret = cos_client.del_folder(request) print 'delete folder ret:', repr(delete_folder_ret)