def _resumable_upload():
self.uploaded_disk_name = 'bakery-' + os.path.basename(self.disk_to_upload) + '-' + \
''.join(random.choices(string.digits, k=6))
AlibabaDisk.iter += 1
LOGGER.info('Upload iteration number %d', AlibabaDisk.iter)
LOGGER.info('Uploading %s as %s', self.disk_to_upload,
self.uploaded_disk_name)
start_time = time.time()
time.sleep(1)
result = False
try:
resumable_store = oss2.resumable.ResumableStore(
root=self.working_dir)
oss2.resumable_upload(self.bucket,
self.uploaded_disk_name,
self.disk_to_upload,
store=resumable_store,
num_threads=number_of_threads)
result = True
except FileNotFoundError as exc:
LOGGER.exception(exc)
raise RuntimeError('Could not find file to upload: {}'.format(
self.disk_to_upload))
except oss2.exceptions.NoSuchUpload as exc:
LOGGER.error('Upload failed. UploadId: %s',
exc.details['UploadId'])
LOGGER.exception(exc)
LOGGER.info('Iteration %d of upload took %d seconds',
AlibabaDisk.iter,
time.time() - start_time)
if not result:
self.upload_cleanup()
return result
def test_progress(self):
stats = {'previous': -1, 'ncalled': 0}
def progress_callback(bytes_consumed, total_bytes):
self.assertTrue(bytes_consumed <= total_bytes)
self.assertTrue(bytes_consumed > stats['previous'])
stats['previous'] = bytes_consumed
stats['ncalled'] += 1
key = random_string(16)
content = random_bytes(5 * 100 * 1024 + 100)
pathname = self._prepare_temp_file(content)
part_size = 100 * 1024
oss2.resumable_upload(self.bucket, key, pathname,
multipart_threshold=200 * 1024,
part_size=part_size,
progress_callback=progress_callback,
num_threads=1)
self.assertEqual(stats['previous'], len(content))
self.assertEqual(stats['ncalled'], oss2.utils.how_many(len(content), part_size) + 1)
stats = {'previous': -1, 'ncalled': 0}
oss2.resumable_upload(self.bucket, key, pathname,
multipart_threshold=len(content) + 100,
progress_callback=progress_callback)
self.assertEqual(stats['previous'], len(content))
self.bucket.delete_object(key)
def upload_files(self, objname, localfile):
oss2.resumable_upload(
self.Bucket,
objname,
localfile,
store=oss2.ResumableStore(
root=current_app.config['UPLOADED_FILE_DEST']))
def __test_interrupt(self, content_size, failed_part_number,
expected_unfinished=0,
modify_record_func=None):
orig_upload_part = oss2.Bucket.upload_part
def upload_part(self, key, upload_id, part_number, data):
if part_number == failed_part_number:
raise RuntimeError
else:
return orig_upload_part(self, key, upload_id, part_number, data)
key = 'resume-' + random_string(32)
content = random_bytes(content_size)
pathname = self._prepare_temp_file(content)
with patch.object(oss2.Bucket, 'upload_part', side_effect=upload_part, autospec=True) as mock_upload_part:
self.assertRaises(RuntimeError, oss2.resumable_upload, self.bucket, key, pathname,
multipart_threshold=0,
part_size=100 * 1024)
if modify_record_func:
modify_record_func(oss2.resumable.make_upload_store(), self.bucket.bucket_name, key, pathname)
oss2.resumable_upload(self.bucket, key, pathname, multipart_threshold=0, part_size=100 * 1024)
self.assertEqual(len(list(oss2.ObjectUploadIterator(self.bucket, key))), expected_unfinished)
def resumableUpload(self, path):
"""断点续传上传
Args:
path (TYPE): file abspath
Returns:
TYPE: Description
"""
part_size = os.path.getsize(path) if os.path.getsize(
path) < 1024 * 1024 else os.path.getsize(path) // 10
success = False
retry = 10
while not success and retry > 0:
retry -= 1
try:
oss2.resumable_upload(
self.bucket,
path.rsplit(os.sep, 1)[1],
path,
progress_callback=self.percentage,
# store=oss2.ResumableStore(root='/tmp'),
store=oss2.ResumableStore(root='/tmp' if checkOS() ==
'linux' else config.BASE_DIR),
multipart_threshold=1024 * 1024,
part_size=part_size,
num_threads=4)
success = True
return True
except oss2.exceptions.RequestError as e:
log.warn('上传失败，即将进行重试')
time.sleep(2)
continue
return False
def uploadData(self, localPath, uploadName):
print 'uploading data'
req = request.Request(host=self.__host, protocol=constant.HTTP, url='/credential/upload/data/' + uploadName, method="GET",
time_out=3000)
a, b, rr = self.cli.execute(req)
try:
res = json.loads(rr)
except:
print a, b, rr
return
if res['success'] != True:
print res
return
token = res['credential']
auth = oss2.StsAuth(token['AccessKeyId'], token['AccessKeySecret'], token['SecurityToken'])
bucket = oss2.Bucket(auth, self.__endpoint, self.__bucket)
oss2.resumable_upload(bucket, res['path'], localPath)
print 'upload complete'
def uploadFiles(bucket):
""" uploadFiles
Upload FLAGS.files to the oss
"""
start_time = time.time()
for tmp_file in FLAGS.files:
if not os.path.exists(tmp_file):
print("File {0} is not exists!".format(tmp_file))
else:
print("Will upload {0} to the oss!".format(tmp_file))
tmp_time = time.time()
# cut the file name
filename = tmp_file[tmp_file.rfind("/") + 1:len(tmp_file)]
ossFilename = os.path.join(FLAGS.prefix, filename)
oss2.resumable_upload(bucket,
ossFilename,
tmp_file,
progress_callback=percentage)
print("\nFile {0} -> {1} uploads finished, cost {2} Sec.".format(
tmp_file, ossFilename,
time.time() - tmp_time))
print("All upload tasks have finished!")
print("Cost {0} Sec.".format(time.time() - start_time))
def vodeo_update(video_url, video_full_path):
retry_time = 1
result = False
myLog.info(u"文件:%s开始上传,oss对象:%s" % (video_full_path, video_url))
global bucket_video
while retry_time <= setting.RETRY_TIMES:
if not bucket_video:
bucket_video = get_bucket_video(video_url)
try:
resumable_upload(bucket_video,
video_url,
video_full_path,
store=ResumableStore(root=setting.UPLOAD_CACHE),
multipart_threshold=setting.PART_SIZE,
part_size=setting.PART_SIZE,
num_threads=4)
myLog.info(u"文件:%s上传成功,oss对象:%s" % (video_full_path, video_url))
result = True
break
except Exception as e:
bucket_video = None
sleep_seconds = setting.WRONG_SLEEP**retry_time
myLog.warning(u"文件:%s上传oss:%s失败,%s后重试" %
(video_full_path, video_url, sleep_seconds))
myLog.error(e)
time.sleep(sleep_seconds)
retry_time += 1
return result
def test_progress(self):
stats = {'previous': -1}
def progress_callback(bytes_consumed, total_bytes):
self.assertTrue(bytes_consumed <= total_bytes)
self.assertTrue(bytes_consumed > stats['previous'])
stats['previous'] = bytes_consumed
key = random_string(16)
content = random_bytes(5 * 100 * 1024 + 100)
pathname = self._prepare_temp_file(content)
oss2.resumable_upload(self.bucket,
key,
pathname,
multipart_threshold=200 * 1024,
part_size=100 * 1024,
progress_callback=progress_callback)
self.assertEqual(stats['previous'], len(content))
stats = {'previous': -1}
oss2.resumable_upload(self.bucket,
key,
pathname,
multipart_threshold=len(content) + 100,
progress_callback=progress_callback)
self.assertEqual(stats['previous'], len(content))
self.bucket.delete_object(key)
def test_progress(self):
bucket = random.choice([self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket])
stats = {'previous': -1, 'ncalled': 0}
def progress_callback(bytes_consumed, total_bytes):
self.assertTrue(bytes_consumed <= total_bytes)
self.assertTrue(bytes_consumed > stats['previous'])
stats['previous'] = bytes_consumed
stats['ncalled'] += 1
key = random_string(16)
content = random_bytes(5 * 100 * 1024 + 100)
pathname = self._prepare_temp_file(content)
part_size = 100 * 1024
oss2.resumable_upload(bucket, key, pathname,
multipart_threshold=200 * 1024,
part_size=part_size,
progress_callback=progress_callback,
num_threads=1)
self.assertEqual(stats['previous'], len(content))
self.assertEqual(stats['ncalled'], oss2.utils.how_many(len(content), part_size) + 1)
stats = {'previous': -1, 'ncalled': 0}
oss2.resumable_upload(bucket, key, pathname,
multipart_threshold=len(content) + 100,
progress_callback=progress_callback)
self.assertEqual(stats['previous'], len(content))
bucket.delete_object(key)
def test_upload_sequenial(self):
endpoint = "http://oss-cn-shanghai.aliyuncs.com"
auth = oss2.Auth(OSS_ID, OSS_SECRET)
bucket_name = OSS_BUCKET + "-test-upload-sequential"
bucket = oss2.Bucket(auth, endpoint, bucket_name)
bucket.create_bucket()
key = random_string(16)
content = random_bytes(5 * 100 * 1024)
pathname = self._prepare_temp_file(content)
oss2.resumable_upload(bucket,
key,
pathname,
multipart_threshold=200 * 1024,
part_size=None)
result = bucket.get_object(key)
self.assertIsNone(result.resp.headers.get('Content-MD5'))
params = {'sequential': ''}
oss2.resumable_upload(bucket,
key,
pathname,
multipart_threshold=200 * 1024,
part_size=None,
params=params)
result = bucket.get_object(key)
self.assertIsNotNone(result.resp.headers.get('Content-MD5'))
bucket.delete_object(key)
bucket.delete_bucket()
def put_obj(self,
key,
src,
use_resume=True,
part_size=(20 * 1024 * 1024),
num_threads=4):
"""
put file to oss
:param key:
:param src:
"""
# use resume
try:
if use_resume:
oss2.resumable_upload(self.bucket,
key,
src,
store=oss2.ResumableStore(root='/tmp'),
multipart_threshold=100 * 1024,
part_size=part_size,
num_threads=num_threads)
else:
self.bucket.put_object_from_file(key, src)
except Exception as ex:
print ex.message
def _do(self, job):
config = Config()
if job.action == _Job.PUSH:
encode_md5 = base64.b64encode(bytearray.fromhex(job.md5)).decode()
headers = {
"Content-MD5": encode_md5,
snapshot.AliOssSnapshot.meta_md5: job.md5
}
try:
oss2.resumable_upload(
self.target_snapshot.bucket,
job.target,
job.src,
headers=headers,
store=oss2.ResumableStore(root=config.cache_dir),
multipart_threshold=config.multipart_threshold,
part_size=config.multipart_threshold,
num_threads=config.num_threads)
except oss2.exceptions.InvalidDigest:
job.info = "md5 mismatch"
raise JobError
elif job.action == _Job.REMOVE:
self.target_snapshot.bucket.delete_object(job.target)
def __test_resume(self, content_size, uploaded_parts, expected_unfinished=0):
part_size = 100 * 1024
num_parts = (content_size + part_size - 1) // part_size
key = 'resume-' + random_string(32)
content = random_bytes(content_size)
pathname = self._prepare_temp_file(content)
upload_id = self.bucket.init_multipart_upload(key).upload_id
for part_number in uploaded_parts:
start = (part_number -1) * part_size
if part_number == num_parts:
end = content_size
else:
end = start + part_size
self.bucket.upload_part(key, upload_id, part_number, content[start:end])
oss2.resumable._rebuild_record(pathname, oss2.resumable.make_upload_store(), self.bucket, key, upload_id, part_size)
oss2.resumable_upload(self.bucket, key, pathname, multipart_threshold=0, part_size=100 * 1024)
result = self.bucket.get_object(key)
self.assertEqual(content, result.read())
self.assertEqual(len(list(oss2.ObjectUploadIterator(self.bucket, key))), expected_unfinished)
self.bucket.delete_object(key)
def uploadFile(self, path, localPath):
oss2.resumable_upload(self._bucket,
path,
localPath,
store=oss2.ResumableDownloadStore(root='./tmp'),
multipart_threshold=20 * 1024 * 1024,
part_size=10 * 1024 * 1024,
num_threads=3)
def multi_upload_obj(self, remote_file, local_file):
oss2.resumable_upload(
self._bucket,
remote_file,
local_file,
store=oss2.ResumableStore(root='/tmp'), #指定保存断点信息的目录
multipart_threshold=100 * 1024, #文件长度大于该值时，则用分片上传
part_size=100 * 1024, #分片大小
num_threads=4) #并发上传的线程数
def test_resumable_upload(self):
small_object = 'requestpayment-test-resumable-upload-small-object'
big_object = 'requestpayment-test-resumable-upload-big-object'
# Create tmp file smaller than multipart_threshold
file_name = self._prepare_temp_file_with_size(150 * 1024)
# Resumale upload small object without payer setting, should be failed.
self.assertRaises(oss2.exceptions.AccessDenied, oss2.resumable_upload, self.payer_bucket, small_object, file_name,
multipart_threshold=(200*1024), num_threads=2, part_size=(100*1024))
# Resumale upload small object with payer setting, should be successful.
headers = dict()
headers[OSS_REQUEST_PAYER] = "requester"
result = oss2.resumable_upload(self.payer_bucket, small_object, file_name,
multipart_threshold=(200*1024), num_threads=2, part_size=(100*1024), headers=headers)
self.assertEqual(result.status, 200)
self.bucket.delete_object(small_object)
# Start big file test
# Create big file bigger than multipart_threshold
file_name = self._prepare_temp_file_with_size(11 *1024 * 1024)
# Resumale upload big object without payer setting, should be failed.
self.assertRaises(oss2.exceptions.AccessDenied, oss2.resumable_upload, self.payer_bucket, big_object, file_name,
multipart_threshold=(200*1024), num_threads=2, part_size=(100*1024))
# Resumale upload big object with payer setting and tagging setting, should be successful.
key1 = 'key1'
value1 = 'value2'
key2 = 'key2'
value2 = 'value2'
tag_str = key1 + '=' + value1
tag_str += '&' + key2 + '=' + value2
headers = dict()
headers[OSS_REQUEST_PAYER] = "requester"
headers[OSS_OBJECT_TAGGING] = tag_str
result = oss2.resumable_upload(self.payer_bucket, big_object, file_name,
multipart_threshold=(200*1024), num_threads=2, part_size=(100*1024), headers=headers)
self.assertEqual(result.status, 200)
# Check object size
head_info = self.bucket.head_object(big_object)
total_size = head_info.content_length
self.assertEqual(total_size, (11 * 1024 * 1024))
# Check tagging
result = self.bucket.get_object_tagging(big_object)
self.assertEqual(2, result.tag_set.len())
tagging_rule = result.tag_set.tagging_rule
self.assertEqual(value1, tagging_rule[key1])
self.assertEqual(value2, tagging_rule[key2])
self.bucket.delete_object(big_object)
def uploadFile(file):
fileRemote = os.path.split(file)[1]
print("=====上传 %s 到阿里云 =====" % fileRemote)
# 带进度条的断点续传
oss2.resumable_upload(bucket, fileRemote, file,
multipart_threshold=200*1024,
part_size=100*1024,
num_threads=4,
progress_callback=percentage)
def __test_resume(self,
content_size,
uploaded_parts,
expected_unfinished=0):
bucket = random.choice(
[self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket])
part_size = 100 * 1024
num_parts = (content_size + part_size - 1) // part_size
key = 'resume-' + random_string(32)
content = random_bytes(content_size)
encryption_flag = isinstance(bucket, oss2.CryptoBucket)
context = None
pathname = self._prepare_temp_file(content)
if encryption_flag:
context = models.MultipartUploadCryptoContext(
content_size, part_size)
upload_id = bucket.init_multipart_upload(
key, upload_context=context).upload_id
else:
upload_id = bucket.init_multipart_upload(key).upload_id
for part_number in uploaded_parts:
start = (part_number - 1) * part_size
if part_number == num_parts:
end = content_size
else:
end = start + part_size
if encryption_flag:
bucket.upload_part(key,
upload_id,
part_number,
content[start:end],
upload_context=context)
else:
bucket.upload_part(key, upload_id, part_number,
content[start:end])
self._rebuild_record(pathname, oss2.resumable.make_upload_store(),
bucket, key, upload_id, part_size, context)
oss2.resumable_upload(bucket,
key,
pathname,
multipart_threshold=0,
part_size=100 * 1024)
result = bucket.get_object(key)
self.assertEqual(content, result.read())
self.assertEqual(
len(list(oss2.ObjectUploadIterator(self.bucket, key))),
expected_unfinished)
bucket.delete_object(key)
def upload(host, user, password, port, my_conf):
abc = lib_innodb_backup.backup(host, user, password, port, my_conf)
auth = oss2.Auth('LTAIQFvh36NyxuxI', 'M6Au2n69pz7mZFVrICKaQaFKxTv2do')
# service = oss2.Service(auth, 'oss-cn-beijing.aliyuncs.com')
# print([b.name for b in oss2.BucketIterator(service)])
ticks = time.strftime("%Y%d%m_%H%M")
bucket = oss2.Bucket(auth, 'oss-cn-beijing.aliyuncs.com',
'tplinuxmysqlbackup')
oss2.resumable_upload(bucket, ticks + '.tar.gz', "%s.tar.gz" % abc)
def uploadFile(self):
try:
oss2.resumable_upload(self.ossBucket,
self.objectname,
self.file,
multipart_threshold=100 * 1024,
num_threads=4,
progress_callback=self.percentage)
except Exception as e:
print(e)
def upload_war(warfilepath):
auth = oss2.Auth(accessKeyId, accessKeySecret)
bucket = oss2.Bucket(auth, ossendpoint, ossbucketname)
oss2.resumable_upload(bucket, os.path.basename(warfilepath), warfilepath)
warurl = "https://" + ossbucketname + "." + ossendpoint + "/" + os.path.basename(
warfilepath)
return warurl
def uploadFile(file):
fileRemote = os.path.split(file)[1]
print("=====上传 %s 到阿里云 =====" % fileRemote)
# 带进度条的断点续传
oss2.resumable_upload(bucket,
fileRemote,
file,
multipart_threshold=200 * 1024,
part_size=100 * 1024,
num_threads=4,
progress_callback=percentage)
def upload(self,file):
with open(file,'r+') as f:
lines = f.readlines()
for line in lines:
line1 = line.strip('\n')
remote_file = line1.split('/data1/jenkins_dir/')
oss2.resumable_upload(bucket, remote_file[1], line1,
store=oss2.ResumableStore(root='/tmp'),
multipart_threshold=100*1024,
part_size=100*1024,
num_threads=4)
def test_concurrency(self):
bucket = random.choice([self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket])
key = random_string(16)
content = random_bytes(64 * 100 * 1024)
pathname = self._prepare_temp_file(content)
oss2.resumable_upload(bucket, key, pathname, multipart_threshold=200 * 1024, part_size=100 * 1024,
num_threads=8)
result = bucket.get_object(key)
self.assertEqual(content, result.read())
self.assertEqual(result.headers['x-oss-object-type'], 'Multipart')
def up(img_path_, remote_img_loc_, img_):
if not bucket.object_exists(remote_img_loc_): # 判断远端文件是否存在
oss2.resumable_upload(bucket,
remote_img_loc_,
img_path_,
multipart_threshold=200 * 1024,
part_size=100 * 1024,
num_threads=3,
progress_callback=percentage)
print(', ' + img_ + ' is successfully uploaded.')
else:
print(img_ + " already exists, ignore it.")
def upload_to_oss(path):
config = configparser.ConfigParser()
config.read(os.path.expanduser("~/.ossutilconfig"))
OSS_ENDPOINT = config["Credentials"]["endpoint"]
OSS_ACCESS_KEY_ID = config["Credentials"]["accessKeyID"]
OSS_ACCESS_KEY_SECRET = config["Credentials"]["accessKeySecret"]
OSS_BUCKET = "kolibri"
oss_auth = oss2.Auth(OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET)
bucket = oss2.Bucket(oss_auth, OSS_ENDPOINT, OSS_BUCKET)
filename = os.path.split(path)[1]
oss2.resumable_upload(bucket, filename, path)
def resumable(self, cloud_file, file_to_upload):
print "开始断点续传%s" % (file_to_upload)
startTime = time.time()
bucket = self.__connect_oss()
oss2.resumable_upload(bucket, cloud_file, file_to_upload,
store=oss2.ResumableStore(root='/tmp'),
multipart_threshold=self.__file_critical_size,
part_size=self.__file_chunk_size,
num_threads=10)
endTime = time.time()
spendTime = endTime - startTime
print "Upload file spend %f second." % (spendTime)
def test_upload_small(self):
key = random_string(16)
content = random_bytes(100)
pathname = self._prepare_temp_file(content)
oss2.resumable_upload(self.bucket, key, pathname)
result = self.bucket.get_object(key)
self.assertEqual(content, result.read())
self.assertEqual(result.headers['x-oss-object-type'], 'Normal')
self.bucket.delete_object(key)
def test_upload_small(self):
key = random_string(16)
content = random_bytes(100)
pathname = self._prepare_temp_file(content)
oss2.resumable_upload(self.bucket, key, pathname)
result = self.bucket.get_object(key)
self.assertEqual(content, result.read())
self.assertEqual(result.headers['x-oss-object-type'], 'Normal')
self.bucket.delete_object(key)
def upload_to_oss(local_path, endpoint, remote_path):
bucket = get_bucket(endpoint)
local_path = os.path.abspath(local_path)
if os.path.isdir(local_path):
for top, dirs, files in os.walk(local_path):
for file in files:
local_file = os.path.join(top, file)
remote_file = local_file.replace(local_path, remote_path)
print("Uploading {0}...".format(local_file))
oss2.resumable_upload(bucket, remote_file, local_file)
else:
print("Uploading {0}...".format(local_path))
oss2.resumable_upload(bucket, remote_path, local_path)
def test_upload_large(self):
key = random_string(16)
content = random_bytes(5 * 100 * 1024)
pathname = self._prepare_temp_file(content)
oss2.resumable_upload(self.bucket, key, pathname, multipart_threshold=200 * 1024, part_size=None)
result = self.bucket.get_object(key)
self.assertEqual(content, result.read())
self.assertEqual(result.headers['x-oss-object-type'], 'Multipart')
self.bucket.delete_object(key)
def test_concurrency(self):
key = random_string(16)
content = random_bytes(64 * 100 * 1024)
pathname = self._prepare_temp_file(content)
oss2.resumable_upload(self.bucket, key, pathname,
multipart_threshold=200 * 1024,
part_size=100*1024,
num_threads=8)
result = self.bucket.get_object(key)
self.assertEqual(content, result.read())
self.assertEqual(result.headers['x-oss-object-type'], 'Multipart')
def _put(self, source_path, remote_filename):
key_name = self.key_prefix + remote_filename
# Default multipart parallel threads number is 5.
multipart_parallel = 5
if vars().has_key('globals.alicloud_multipart_upload_threads'):
if globals.alicloud_multipart_upload_threads >= 5:
multipart_parallel = globals.alicloud_multipart_upload_threads
import oss2
oss2.resumable_upload(self.bucket,
key_name,
source_path.name,
num_threads=multipart_parallel)
def upload(img_path_, remote_img_ref, img_full_filename):
# 上传过程的函数，使用断点续传
if not bucket.object_exists(remote_img_ref): # 判断远端文件是否存在
oss2.resumable_upload(
bucket,
remote_img_ref,
img_path_,
multipart_threshold=200 * 1024,
part_size=100 * 1024,
num_threads=3,
progress_callback=percentage)
print(', ' + img_full_filename + ' is successfully uploaded.')
else:
print(img_full_filename + " already exists, ignore it.")
def upload_oss(path, name):
"""
:param path:
:param name: name is filename stored on OSS, which known as 'key' in oss2.
:return:
"""
# KNOWN ISSUE: oss2 has a bug in oss2.resumable_upload() which not support non-ascii character
path = to_unicode_or_bust(path) # support support non-ascii character, such as path name in Chinese Language
bucket = oss2.Bucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name)
if not os.path.exists(path): # need <type 'unicode'>
raise OSError("cannot access '%s': No such file or directory" % to_str_or_bust(path))
else:
logger.info("file \'%s\' selected." % path)
if name:
# OSS中的目录/文件夹概念
# https://help.aliyun.com/knowledge_detail/39527.html
key = name
else:
key = os.path.basename(path)
path = to_str_or_bust(path)
key = to_str_or_bust(key)
logger.info("ready for uploading file to oss")
result = oss2.resumable_upload(bucket, key, path, multipart_threshold=10 * 1024 * 1024)
oss_obj = bucket.get_object(key)
logger.info(" ".join((oss_obj.request_id, str(oss_obj.status))))
logger.info("file \'%s\' uploaded." % path)
return result
def test_upload_large(self):
key = random_string(16)
content = random_bytes(5 * 100 * 1024)
pathname = self._prepare_temp_file(content)
result = oss2.resumable_upload(self.bucket, key, pathname, multipart_threshold=200 * 1024, part_size=None)
self.assertTrue(result is not None)
self.assertTrue(result.etag is not None)
self.assertTrue(result.request_id is not None)
result = self.bucket.get_object(key)
self.assertEqual(content, result.read())
self.assertEqual(result.headers['x-oss-object-type'], 'Multipart')
self.bucket.delete_object(key)
# 上传分片
for i in range(3):
result = bucket.upload_part(key, upload_id, i+1, content, progress_callback=percentage)
parts.append(oss2.models.PartInfo(i+1, result.etag))
# 完成上传并回调
result = bucket.complete_multipart_upload(key, upload_id, parts)
"""
断点续传上传
"""
# 带进度条的断点续传
pathname = _prepare_temp_file(content)
oss2.resumable_upload(bucket, key, pathname,
multipart_threshold=200*1024,
part_size=100*1024,
num_threads=3,
progress_callback=percentage)
"""
文件下载
"""
# 带进度条的下载
result = bucket.get_object(key, progress_callback=percentage)
content_got = b''
for chunk in result:
content_got += chunk
assert content == content_got
"""
范围下载
def resumable_upload(bucket, object_key, local_file_path):
# get a file path (string)
oss2.resumable_upload(bucket, object_key, local_file_path, progress_callback=percentage, num_threads=4)
from __future__ import print_function
import os, sys
import oss2
#
# 百分比显示回调函数
#
def percentage(consumed_bytes, total_bytes):
if total_bytes:
rate = int(100 * (float(consumed_bytes) / float(total_bytes)))
print('\r{0}% '.format(rate), end=filePath)
sys.stdout.flush()
# 脚本需要传入5个参数
if ( len(sys.argv) > 5 ):
AccessKeyId = sys.argv[1]
AccessKeySecret = sys.argv[2]
Endpoint = sys.argv[3]
Bucket = sys.argv[4]
filePath = sys.argv[5]
fileName = filePath.split("/")[-1]
else:
print("Example: %s AccessKeyId AccessKeySecret Endpoint Bucket /data/backup.zip" % sys.argv[0])
exit()
# OSS认证并开始上传
auth = oss2.Auth(AccessKeyId , AccessKeySecret)
bucket = oss2.Bucket(auth, Endpoint, Bucket)
oss2.resumable_upload(bucket, fileName, filePath, progress_callback=percentage)
print('\rUpload %s to OSS Success!' % filePath)
bucket = oss2.Bucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name)
def random_string(n):
return ''.join(random.choice(string.ascii_lowercase) for i in range(n))
# 生成一个本地文件用于测试。文件内容是bytes类型。
filename = random_string(32) + '.txt'
content = oss2.to_bytes(random_string(1024 * 1024))
with open(filename, 'wb') as fileobj:
fileobj.write(content)
# 断点续传一：因为文件比较小（小于oss2.defaults.multipart_threshold），
# 所以实际上用的是oss2.Bucket.put_object
oss2.resumable_upload(bucket, 'remote-normal.txt', filename)
# 断点续传二：为了展示的需要，我们指定multipart_threshold可选参数，确保使用分片上传
oss2.resumable_upload(bucket, 'remote-multipart.txt', filename, multipart_threshold=100 * 1024)
# 也可以直接调用分片上传接口。
# 首先可以用帮助函数设定分片大小，设我们期望的分片大小为128KB
total_size = os.path.getsize(filename)
part_size = oss2.determine_part_size(total_size, preferred_size=128 * 1024)
# 初始化分片上传，得到Upload ID。接下来的接口都要用到这个Upload ID。
key = 'remote-multipart2.txt'
upload_id = bucket.init_multipart_upload(key).upload_id
# 逐个上传分片
