def __test_normal(self, file_size):
bucket = random.choice(
[self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket])
key, filename, content = self.__prepare(bucket, file_size)
oss2.resumable_download(bucket, key, filename)
self.assertFileContent(filename, content)
def downloadFiles(bucket):
""" downloadFiles
download FLAGS.files on the oss
"""
if not os.path.exists(FLAGS.outputPath):
os.makedirs(FLAGS.outputPath)
print("The floder {0} is not existed, will creat it".format(
FLAGS.outputPath))
start_time = time.time()
for tmp_file in FLAGS.files:
if not bucket.object_exists(tmp_file):
print("File {0} is not on the OSS!".format(tmp_file))
else:
print("Will download {0} !".format(tmp_file))
tmp_time = time.time()
# cut the file name
filename = tmp_file[tmp_file.rfind("/") + 1:len(tmp_file)]
localFilename = os.path.join(FLAGS.outputPath, filename)
# bucket.get_object_to_file(
oss2.resumable_download(bucket,
tmp_file,
localFilename,
progress_callback=percentage)
print("\nFile {0} -> {1} downloads finished, cost {2} Sec.".format(
tmp_file, localFilename,
time.time() - tmp_time))
print("All download tasks have finished!")
print("Cost {0} Sec.".format(time.time() - start_time))
def download_fonts(self, keyword='', pref=''):
for fonts_bucket in iter(self.get_font_bucket()):
# get bucket region
fonts_bucket_info = fonts_bucket.get_bucket_info()
region = fonts_bucket_info.location.split('-')[-1]
# get font bucket sotorage dir
fonts_dir = config.ali_fonts_bucket[region]['font_dir']
bucket_name = fonts_bucket.bucket_name
# oss2.ObjectIteratorr用于遍历文件。
oss_object_list = oss2.ObjectIterator(fonts_bucket)
for font_file in oss_object_list:
file_name = font_file.key.split(fonts_dir)[-1]
if file_name.endswith('tf') and keyword in file_name and file_name.startswith(pref) and fonts_dir in font_file.key:
print('fonts %s matched for download in bucket %s' % (font_file.key, fonts_bucket.bucket_name))
self.logger.info(
'fonts %s matched for download in bucket %s' % (font_file.key, fonts_bucket.bucket_name))
try:
oss2.resumable_download(fonts_bucket, font_file, '../downloads/' + font_file,
part_size=100 * 1024,
num_threads=3,
progress_callback=self.percentage,
store=oss2.ResumableDownloadStore(root='./tmp_files/downloads'))
except oss2.exceptions.NotFound as en:
self.logger.exception('Font %s not found while download fonts' % font_file)
except Exception as e:
self.logger.exception('Exception catched while download fonts %s: %s' % (font_file, e))
else:
# print('fonts %s not matched for download in bucket %s' % (file_name, fonts_bucket.bucket_name))
self.logger.debug('fonts %s not matched for download in bucket %s' % (file_name, fonts_bucket.bucket_name))
def __test_resume(self, file_size, failed_parts, modify_func_record=None):
total = NonlocalObject(0)
orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part
def mock_download_part(self, part, failed_parts=None):
if part.part_number in failed_parts:
raise RuntimeError("Fail download_part for part: {0}".format(part.part_number))
else:
total.var += 1
orig_download_part(self, part)
key, filename, content = self.__prepare(file_size)
with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part',
side_effect=partial(mock_download_part, failed_parts=failed_parts),
autospec=True):
self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename)
store = oss2.resumable.make_download_store()
store_key = store.make_store_key(self.bucket.bucket_name, key, os.path.abspath(filename))
record = store.get(store_key)
tmp_file = filename + record['tmp_suffix']
self.assertTrue(os.path.exists(tmp_file))
self.assertTrue(not os.path.exists(filename))
with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part',
side_effect=partial(mock_download_part, failed_parts=[]),
autospec=True):
oss2.resumable_download(self.bucket, key, filename)
self.assertEqual(total.var, oss2.utils.how_many(file_size, oss2.defaults.multiget_part_size))
self.assertTrue(not os.path.exists(tmp_file))
self.assertFileContent(filename, content)
def downloadfile(self, rpath, lpath, **kwargs):
"""
:param rpath: 远程文件路径
:param lpath: 本地文件路径
:return:
"""
oss2.resumable_download(self.bucket, rpath, lpath, **kwargs)
def test_tmp_file_removed(self):
oss2.defaults.multiget_threshold = 1
oss2.defaults.multiget_part_size = 100
oss2.defaults.multiget_num_threads = 5
orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part
file_size = 123 * 3 + 1
key, filename, content = self.__prepare(file_size)
context = {}
def mock_download_part(downloader, part, part_number=None):
if part.part_number == part_number:
r = self.__record(key, filename)
context['tmpfile'] = filename + r['tmp_suffix']
raise RuntimeError("Fail download_part for part: {0}".format(part_number))
else:
orig_download_part(downloader, part)
with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part',
side_effect=partial(mock_download_part, part_number=2),
autospec=True):
self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename)
os.remove(context['tmpfile'])
oss2.resumable_download(self.bucket, key, filename)
self.assertFileContent(filename, content)
def get_obj(self,
key,
dst,
use_resume=True,
part_size=(20 * 1024 * 1024),
num_threads=4):
"""
get files from oss,
:param key: oss key
:param dst: The path to save obj.
:return obj save path at last
"""
try:
if self.is_public_net:
raise Exception('Do not download from public')
if use_resume:
oss2.resumable_download(
self.bucket,
key,
dst,
store=oss2.ResumableDownloadStore(root='/tmp'),
multiget_threshold=20 * 1024 * 1024,
part_size=part_size,
num_threads=num_threads)
else:
self.bucket.get_object_to_file(key, dst)
return dst
except Exception as ex:
print ex.message
return None
def __test_resume(self, file_size, failed_parts, modify_func_record=None):
total = NonlocalObject(0)
orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part
def mock_download_part(self, part, failed_parts=None):
if part.part_number in failed_parts:
raise RuntimeError("Fail download_part for part: {0}".format(part.part_number))
else:
total.var += 1
orig_download_part(self, part)
key, filename, content = self.__prepare(file_size)
with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part',
side_effect=partial(mock_download_part, failed_parts=failed_parts),
autospec=True):
self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename)
store = oss2.resumable.make_download_store()
store_key = store.make_store_key(self.bucket.bucket_name, key, os.path.abspath(filename))
record = store.get(store_key)
tmp_file = filename + record['tmp_suffix']
self.assertTrue(os.path.exists(tmp_file))
self.assertTrue(not os.path.exists(filename))
with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part',
side_effect=partial(mock_download_part, failed_parts=[]),
autospec=True):
oss2.resumable_download(self.bucket, key, filename)
self.assertEqual(total.var, oss2.utils.how_many(file_size, oss2.defaults.multiget_part_size))
self.assertTrue(not os.path.exists(tmp_file))
self.assertFileContent(filename, content)
def test_tmp_file_removed(self):
oss2.defaults.multiget_threshold = 1
oss2.defaults.multiget_part_size = 100
oss2.defaults.multiget_num_threads = 5
orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part
file_size = 123 * 3 + 1
key, filename, content = self.__prepare(file_size)
context = {}
def mock_download_part(downloader, part, part_number=None):
if part.part_number == part_number:
r = self.__record(key, filename)
context['tmpfile'] = filename + r['tmp_suffix']
raise RuntimeError(
"Fail download_part for part: {0}".format(part_number))
else:
orig_download_part(downloader, part)
with patch.object(oss2.resumable._ResumableDownloader,
'_ResumableDownloader__download_part',
side_effect=partial(mock_download_part,
part_number=2),
autospec=True):
self.assertRaises(RuntimeError, oss2.resumable_download,
self.bucket, key, filename)
os.remove(context['tmpfile'])
oss2.resumable_download(self.bucket, key, filename)
self.assertFileContent(filename, content)
def test_progress(self):
bucket = random.choice(
[self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket])
oss2.defaults.multiget_threshold = 1
oss2.defaults.multiget_part_size = 100
oss2.defaults.multiget_num_threads = 1
oss2.defaults.min_part_size = 100
stats = {'previous': -1, 'called': 0}
def progress_callback(bytes_consumed, total_bytes):
self.assertTrue(bytes_consumed <= total_bytes)
self.assertTrue(bytes_consumed > stats['previous'])
stats['previous'] = bytes_consumed
stats['called'] += 1
file_size = 100 * 5 + 1
key, filename, content = self.__prepare(bucket, file_size)
oss2.resumable_download(bucket,
key,
filename,
progress_callback=progress_callback)
self.assertEqual(stats['previous'], file_size)
self.assertEqual(
stats['called'],
oss2.utils.how_many(file_size, oss2.defaults.multiget_part_size) +
1)
def download(self, ossObject, loaclFile):
oss2.resumable_download(self.bucket, ossObject, loaclFile,
store=oss2.ResumableDownloadStore(root=os.path.dirname(loaclFile)),
multiget_threshold=1 * 1024,
part_size=10 * 1024 * 1024,
num_threads=3,
progress_callback= self.percentage
)
def download(file, local_name):
"""resume download orc file from oss"""
logger.info(f'start download oss file {file}....')
try:
oss2.resumable_download(oss_bucket, file, local_name, num_threads=3)
except Exception as e:
logger.error(e)
download(oss_bucket, local_name)
def downloadFile(self, path, localPath):
oss2.resumable_download(
self._bucket,
path,
localPath,
store=oss2.ResumableDownloadStore(root='./tmp'),
multiget_threshold=20 * 1024 * 1024,
part_size=10 * 1024 * 1024,
num_threads=3)
def test_resume_rename_failed(self):
bucket = random.choice(
[self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket])
size = 500 * 10
part_size = 499
oss2.defaults.multiget_threshold = 1
oss2.defaults.multiget_part_size = part_size
oss2.defaults.multiget_num_threads = 3
oss2.defaults.min_part_size = 499
key, filename, content = self.__prepare(bucket, size)
with patch.object(os,
'rename',
side_effect=RuntimeError(),
autospec=True):
self.assertRaises(RuntimeError, oss2.resumable_download, bucket,
key, filename)
r = self.__record(key, filename)
# assert record fields are valid
head_object_result = bucket.head_object(key)
self.assertEqual(r['size'], size)
self.assertEqual(r['mtime'], head_object_result.last_modified)
self.assertEqual(r['etag'], head_object_result.etag)
self.assertEqual(r['bucket'], bucket.bucket_name)
self.assertEqual(r['key'], key)
self.assertEqual(r['part_size'], part_size)
self.assertTrue(os.path.exists(filename + r['tmp_suffix']))
self.assertFileContent(filename + r['tmp_suffix'], content)
self.assertTrue(not os.path.exists(filename))
self.assertEqual(r['file_path'], os.path.abspath(filename))
self.assertEqual(len(r['parts']), oss2.utils.how_many(size, part_size))
parts = sorted(r['parts'], key=lambda p: p['part_number'])
for i, p in enumerate(parts):
self.assertEqual(p['part_number'], i + 1)
self.assertEqual(p['start'], part_size * i)
self.assertEqual(p['end'], min(part_size * (i + 1), size))
with patch.object(oss2.resumable._ResumableDownloader,
'_ResumableDownloader__download_part',
side_effect=RuntimeError(),
autospec=True):
oss2.resumable_download(bucket, key, filename)
self.assertTrue(not os.path.exists(filename + r['tmp_suffix']))
self.assertFileContent(filename, content)
self.assertEqual(self.__record(key, filename), None)
def download(self,url): if bucket.object_exists(url): local_f = "/data1/jenkins_dir/" + url oss2.resumable_download(bucket, url, local_f, store=oss2.ResumableDownloadStore(root='/tmp'), multiget_threshold=20*1024*1024, part_size=10*1024*1024, num_threads=2) return True else: return False
def _download(
self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
):
with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
import oss2
oss2.resumable_download(
self.oss_service,
from_info.path,
to_file,
progress_callback=pbar.update_to,
)
def test_remote_changed_during_download(self):
bucket = random.choice(
[self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket])
oss2.defaults.multiget_threshold = 1
oss2.defaults.multiget_part_size = 100
oss2.defaults.multiget_num_threads = 2
oss2.defaults.min_part_size = 100
orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part
orig_rename = os.rename
file_size = 1000
key, filename, content = self.__prepare(bucket, file_size)
old_context = {}
new_context = {}
# 模拟download某个part的过程中原始文件的内容被修改了
def mock_download_part(downloader, part, part_number=None):
if part.part_number == part_number:
r = self.__record(key, filename)
old_context['tmp_suffix'] = r['tmp_suffix']
old_context['etag'] = r['etag']
old_context['content'] = random_bytes(file_size)
bucket.put_object(key, old_context['content'])
orig_download_part(downloader, part)
# rename之前将新的record获取到
def mock_rename(src, dst):
r = self.__record(key, filename)
new_context['tmp_suffix'] = r['tmp_suffix']
new_context['etag'] = r['etag']
orig_rename(src, dst)
with patch.object(oss2.resumable._ResumableDownloader,
'_ResumableDownloader__download_part',
side_effect=partial(mock_download_part,
part_number=5),
autospec=True):
self.assertRaises(oss2.exceptions.PreconditionFailed,
oss2.resumable_download, bucket, key, filename)
with patch.object(os, 'rename', side_effect=mock_rename):
oss2.resumable_download(bucket, key, filename)
self.assertTrue(new_context['tmp_suffix'] != old_context['tmp_suffix'])
self.assertTrue(new_context['etag'] != old_context['etag'])
def test_two_downloaders(self):
"""两个downloader同时跑，但是store的目录不一样。"""
oss2.defaults.multiget_threshold = 1
oss2.defaults.multiget_part_size = 100
oss2.defaults.multiget_num_threads = 2
store1 = oss2.make_download_store()
store2 = oss2.make_download_store(dir='.another-py-oss-download')
file_size = 1000
key, filename, content = self.__prepare(file_size)
context1a = {}
context1b = {}
context2 = {}
def mock_rename(src, dst, ctx=None, store=None):
r = self.__record(key, filename, store=store)
ctx['tmp_suffix'] = r['tmp_suffix']
ctx['etag'] = r['etag']
ctx['mtime'] = r['mtime']
raise RuntimeError('intentional')
with patch.object(os, 'rename', side_effect=partial(mock_rename, ctx=context1a, store=store1), autospect=True):
self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename, store=store1)
with patch.object(os, 'rename', side_effect=partial(mock_rename, ctx=context1b, store=store1), autospect=True):
self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename, store=store1)
with patch.object(os, 'rename', side_effect=partial(mock_rename, ctx=context2, store=store2), autospect=True):
self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename, store=store2)
self.assertEqual(context1a['tmp_suffix'], context1b['tmp_suffix'])
self.assertEqual(context1a['etag'], context1b['etag'])
self.assertEqual(context1a['mtime'], context1b['mtime'])
self.assertNotEqual(context1a['tmp_suffix'], context2['tmp_suffix'])
self.assertEqual(context1a['etag'], context2['etag'])
self.assertEqual(context1a['mtime'], context2['mtime'])
self.assertTrue(os.path.exists(filename + context1a['tmp_suffix']))
self.assertTrue(os.path.exists(filename + context2['tmp_suffix']))
oss2.resumable_download(self.bucket, key, filename, store=store1)
self.assertTrue(not os.path.exists(filename + context1a['tmp_suffix']))
self.assertTrue(os.path.exists(filename + context2['tmp_suffix']))
oss2.resumable_download(self.bucket, key, filename, store=store2)
self.assertTrue(not os.path.exists(filename + context2['tmp_suffix']))
def test_resumable_download_with_version(self):
from oss2.models import BucketVersioningConfig
from oss2.models import BatchDeleteObjectVersion
from oss2.models import BatchDeleteObjectVersionList
auth = oss2.Auth(OSS_ID, OSS_SECRET)
bucket_name = OSS_BUCKET + "-test-resumable-download-with-version"
bucket = oss2.Bucket(auth, OSS_ENDPOINT, bucket_name)
bucket.create_bucket(oss2.BUCKET_ACL_PRIVATE)
wait_meta_sync()
config = BucketVersioningConfig()
config.status = 'Enabled'
result = bucket.put_bucket_versioning(config)
wait_meta_sync()
self.version_bucket = bucket
content_small = random_bytes(5 * 1024)
result = bucket.put_object("object_small", content_small)
version_small = result.versionid
filename_small = self.random_filename()
result = oss2.resumable_download(bucket, "object_small",
filename_small)
self.assertFileContent(filename_small, content_small)
content_big = random_bytes(5 * 1024 * 1024)
result = bucket.put_object("object_big", content_big)
version_big = result.versionid
filename_big = self.random_filename()
result = oss2.resumable_download(bucket, "object_big", filename_big)
self.assertFileContent(filename_big, content_big)
version_list = BatchDeleteObjectVersionList()
version_list.append(
BatchDeleteObjectVersion("object_small", version_small))
version_list.append(BatchDeleteObjectVersion("object_big",
version_big))
result = bucket.delete_object_versions(version_list)
self.assertTrue(len(result.delete_versions) == 2)
bucket.delete_bucket()
def test_two_downloaders(self):
"""两个downloader同时跑，但是store的目录不一样。"""
oss2.defaults.multiget_threshold = 1
oss2.defaults.multiget_part_size = 100
oss2.defaults.multiget_num_threads = 2
store1 = oss2.make_download_store()
store2 = oss2.make_download_store(dir='.another-py-oss-download')
file_size = 1000
key, filename, content = self.__prepare(file_size)
context1a = {}
context1b = {}
context2 = {}
def mock_rename(src, dst, ctx=None, store=None):
r = self.__record(key, filename, store=store)
ctx['tmp_suffix'] = r['tmp_suffix']
ctx['etag'] = r['etag']
ctx['mtime'] = r['mtime']
raise RuntimeError('intentional')
with patch.object(os, 'rename', side_effect=partial(mock_rename, ctx=context1a, store=store1), autospect=True):
self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename, store=store1)
with patch.object(os, 'rename', side_effect=partial(mock_rename, ctx=context1b, store=store1), autospect=True):
self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename, store=store1)
with patch.object(os, 'rename', side_effect=partial(mock_rename, ctx=context2, store=store2), autospect=True):
self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename, store=store2)
self.assertEqual(context1a['tmp_suffix'], context1b['tmp_suffix'])
self.assertEqual(context1a['etag'], context1b['etag'])
self.assertEqual(context1a['mtime'], context1b['mtime'])
self.assertNotEqual(context1a['tmp_suffix'], context2['tmp_suffix'])
self.assertEqual(context1a['etag'], context2['etag'])
self.assertEqual(context1a['mtime'], context2['mtime'])
self.assertTrue(os.path.exists(filename + context1a['tmp_suffix']))
self.assertTrue(os.path.exists(filename + context2['tmp_suffix']))
oss2.resumable_download(self.bucket, key, filename, store=store1)
self.assertTrue(not os.path.exists(filename + context1a['tmp_suffix']))
self.assertTrue(os.path.exists(filename + context2['tmp_suffix']))
oss2.resumable_download(self.bucket, key, filename, store=store2)
self.assertTrue(not os.path.exists(filename + context2['tmp_suffix']))
def test_resumable_download(self):
key = 'traffic-limit-test-resumble-download-object'
content = b'a' * OBJECT_SIZE_1MB
file_name = key + '.txt'
# Put object
self.bucket.put_object(key, content)
headers = dict()
headers[OSS_TRAFFIC_LIMIT] = str(LIMIT_100KB)
# Resumable download object smaller than multiget_threshold with traffic limit setting.
start_time_sec = int(time.time())
oss2.resumable_download(self.bucket,
key,
file_name,
multiget_threshold=(OBJECT_SIZE_1MB * 2),
num_threads=1,
headers=headers)
self.assertFileContent(file_name, content)
end_time_sec = int(time.time())
os.remove(file_name)
# Calculate expensed time
expense_time_sec = end_time_sec - start_time_sec
# Theoretical time is 1MB/100KB = 10s, set the minimum theoretical time to 10*0.7s
theoretical_exepnse_min = 10 * 0.7
# Compare to minimum theoretical time
self.assertEqual((expense_time_sec > theoretical_exepnse_min), True)
# Resumable download object bigger than multiget_threshold with traffic limit setting.
start_time_sec = int(time.time())
oss2.resumable_download(self.bucket,
key,
file_name,
multiget_threshold=(OBJECT_SIZE_1MB - 1024),
num_threads=1,
headers=headers)
self.assertFileContent(file_name, content)
end_time_sec = int(time.time())
os.remove(file_name)
self.bucket.delete_object(key)
# Calculate expensed time
expense_time_sec = end_time_sec - start_time_sec
# Theoretical time is 1MB/100KB = 10s, set the minimum theoretical time to 10*0.7s
theoretical_exepnse_min = 10 * 0.7
# Compare to minimum theoretical time
self.assertEqual((expense_time_sec > theoretical_exepnse_min), True)
def __test_insane_record(self,
file_size,
modify_record_func,
old_tmp_exists=True):
bucket = random.choice(
[self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket])
orig_rename = os.rename
obj = NonlocalObject({})
key, filename, content = self.__prepare(bucket, file_size)
def mock_rename(src, dst):
obj.var = self.__record(key, filename)
orig_rename(src, dst)
# 这个地方模拟rename报错
with patch.object(os,
'rename',
side_effect=RuntimeError(),
autospec=True):
self.assertRaises(RuntimeError, oss2.resumable_download, bucket,
key, filename)
store = oss2.resumable.make_download_store()
store_key = store.make_store_key(bucket.bucket_name, key,
os.path.abspath(filename))
r = store.get(store_key)
# 修改record文件
modify_record_func(store, store_key, copy.deepcopy(r))
with patch.object(os, 'rename', side_effect=mock_rename,
autospec=True):
oss2.resumable_download(bucket, key, filename)
new_r = obj.var
self.assertTrue(new_r['tmp_suffix'] != r['tmp_suffix'])
self.assertEqual(new_r['size'], r['size'])
self.assertEqual(new_r['mtime'], r['mtime'])
self.assertEqual(new_r['etag'], r['etag'])
self.assertEqual(new_r['part_size'], r['part_size'])
self.assertEqual(os.path.exists(filename + r['tmp_suffix']),
old_tmp_exists)
self.assertTrue(not os.path.exists(filename + new_r['tmp_suffix']))
oss2.utils.silently_remove(filename + r['tmp_suffix'])
def test_download_success(test_bucket_access):
import os
import oss2
bucket = test_bucket_access
key = "test_only.txt"
filename = 'test_download_success.txt'
oss2.resumable_download(bucket,
key,
filename,
multiget_threshold=200 * 1024,
part_size=100 * 1024,
num_threads=3)
assert filename in os.listdir()
os.remove(filename)
def test_resume_rename_failed(self):
size = 500 * 10
part_size = 499
oss2.defaults.multiget_threshold = 1
oss2.defaults.multiget_part_size = part_size
oss2.defaults.multiget_num_threads = 3
key, filename, content = self.__prepare(size)
with patch.object(os, 'rename', side_effect=RuntimeError(), autospec=True):
self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename)
r = self.__record(key, filename)
# assert record fields are valid
head_object_result = self.bucket.head_object(key)
self.assertEqual(r['size'], size)
self.assertEqual(r['mtime'], head_object_result.last_modified)
self.assertEqual(r['etag'], head_object_result.etag)
self.assertEqual(r['bucket'], self.bucket.bucket_name)
self.assertEqual(r['key'], key)
self.assertEqual(r['part_size'], part_size)
self.assertTrue(os.path.exists(filename + r['tmp_suffix']))
self.assertFileContent(filename + r['tmp_suffix'], content)
self.assertTrue(not os.path.exists(filename))
self.assertEqual(r['abspath'], os.path.abspath(filename))
self.assertEqual(len(r['parts']), oss2.utils.how_many(size, part_size))
parts = sorted(r['parts'], key=lambda p: p['part_number'])
for i, p in enumerate(parts):
self.assertEqual(p['part_number'], i+1)
self.assertEqual(p['start'], part_size * i)
self.assertEqual(p['end'], min(part_size*(i+1), size))
with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part',
side_effect=RuntimeError(),
autospec=True):
oss2.resumable_download(self.bucket, key, filename)
self.assertTrue(not os.path.exists(filename + r['tmp_suffix']))
self.assertFileContent(filename, content)
self.assertEqual(self.__record(key, filename), None)
def download_from_oss(local_path, endpoint, remote_path):
bucket = get_bucket(endpoint)
print("Downloading {0}...".format(remote_path))
try:
return resumable_download(bucket, remote_path, local_path)
except (NotFound, NoSuchKey):
# FIXME: 最多只能下载 1000 个文件
remote_files = _list_files(bucket, remote_path, MAX_FILE_COUNT)
for remote_file in remote_files:
rel = os.path.relpath(remote_file, remote_path)
local_file = os.path.join(local_path, rel)
local_file_dir = os.path.dirname(local_file)
os.makedirs(local_file_dir, exist_ok=True)
print("Downloading {0}...".format(remote_file))
resumable_download(bucket, remote_file, local_file)
def download(self, remote_file, local_file):
'''
:param remote_file: 仓库文件名
:param local_file: 本地文件名
:return:
'''
try:
oss2.resumable_download(self.bucket,
remote_file,
local_file,
multiget_threshold=200 * 1024,
part_size=100 * 1024,
num_threads=3,
progress_callback=self.percentage)
except oss2.exceptions.NotFound:
print("下载的文件不存在！")
def test_resumable_down(self):
small_object = 'requestpayment-test-resumable-down-small-object'
content1 = b'a' * (150 * 1024)
big_object = 'requestpayment-test-resumable-down-big-object'
content2 = b'a' * (500 * 1024)
file_name = small_object + '.txt'
self.bucket.put_object(small_object, content1)
self.bucket.put_object(big_object, content2)
# Resumale down small object without payer setting, should be failed.
self.assertRaises(oss2.exceptions.ServerError, oss2.resumable_download, self.payer_bucket, small_object, file_name,
multiget_threshold=(200*1024), num_threads=2, part_size=(1024*1024))
# Resumale down small object with payer setting, should be successful.
headers = dict()
headers[OSS_REQUEST_PAYER] = "requester"
oss2.resumable_download(self.payer_bucket, small_object, file_name,
multiget_threshold=(200*1024), num_threads=2, part_size=(1024*1024), headers=headers)
# Check file size
file_size = os.stat(file_name).st_size
self.assertEqual(file_size, (150*1024))
os.remove(file_name)
self.bucket.delete_object(small_object)
file_name = big_object + '.txt'
# Resumale down big object without payer setting, should be failed.
self.assertRaises(oss2.exceptions.ServerError, oss2.resumable_download, self.payer_bucket, big_object, file_name,
multiget_threshold=(200*1024), num_threads=2, part_size=(1024*1024))
# Resumale down big object with payer setting, should be successful.
headers = dict()
headers[OSS_REQUEST_PAYER] = "requester"
oss2.resumable_download(self.payer_bucket, big_object, file_name,
multiget_threshold=(200*1024), num_threads=2, part_size=(1024*1024), headers=headers)
# Check file size
file_size = os.stat(file_name).st_size
self.assertEqual(file_size, (500*1024))
os.remove(file_name)
self.bucket.delete_object(big_object)
def download_war(dm_name, version, redis_key):
#下载对应项目的最新war包
try:
#包名需要规范
Redis.lpush(
redis_key,
'%s-%s.war package download from oss ......' % (dm_name, version))
auth = oss2.Auth(oss_id, oss_key)
bucket = oss2.Bucket(auth, 'oss-cn-beijing.aliyuncs.com', 'mojiops')
file_war = None
for obj in oss2.ObjectIterator(bucket, prefix='tomcat-'):
if '%s-%s' % (dm_name, version) in obj.key:
file_war = obj.key
break
if not os.path.exists('%s/%s' % (dockerfile_path, dm_name)):
os.mkdir('%s/%s' % (dockerfile_path, dm_name))
if file_war:
object = file_war.split('/')[-1]
project_path = '%s/%s/%s.war' % (dockerfile_path, dm_name,
dm_name.split('-')[0])
if os.path.exists(project_path):
os.remove(project_path)
#尝试3次下载
for i in range(3):
try:
oss2.resumable_download(bucket, file_war, project_path)
except Exception as e:
logging.error(e)
else:
if os.path.exists(project_path):
#生成dockerfile文件
#
Redis.lpush(redis_key,
'%s package download success!' % object)
return object
else:
Redis.lpush(redis_key, '%s package not fond!' % object)
return None
if not os.path.exists(project_path):
Redis.lpush(redis_key, '%s package download fail!' % object)
return None
except Exception as e:
logging.error(e)
Redis.lpush(redis_key, '%s package download fail!' % object)
return None
def test_remote_changed_during_download(self):
oss2.defaults.multiget_threshold = 1
oss2.defaults.multiget_part_size = 100
oss2.defaults.multiget_num_threads = 2
orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part
orig_rename = os.rename
file_size = 1000
key, filename, content = self.__prepare(file_size)
old_context = {}
new_context = {}
def mock_download_part(downloader, part, part_number=None):
if part.part_number == part_number:
r = self.__record(key, filename)
old_context['tmp_suffix'] = r['tmp_suffix']
old_context['etag'] = r['etag']
old_context['content'] = random_bytes(file_size)
self.bucket.put_object(key, old_context['content'])
orig_download_part(downloader, part)
def mock_rename(src, dst):
r = self.__record(key, filename)
new_context['tmp_suffix'] = r['tmp_suffix']
new_context['etag'] = r['etag']
orig_rename(src, dst)
with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part',
side_effect=partial(mock_download_part, part_number=5),
autospec=True):
self.assertRaises(oss2.exceptions.PreconditionFailed, oss2.resumable_download, self.bucket, key, filename)
with patch.object(os, 'rename', side_effect=mock_rename):
oss2.resumable_download(self.bucket, key, filename)
self.assertTrue(new_context['tmp_suffix'] != old_context['tmp_suffix'])
self.assertTrue(new_context['etag'] != old_context['etag'])
def OssDownload():
accessKeyId = sys.argv[1]
accessSecret = sys.argv[2]
EndPoint = sys.argv[3]
Bucket = sys.argv[4]
RemoteFile = sys.argv[5]
LocalFile = sys.argv[6]
auth = oss2.Auth(accessKeyId, accessSecret)
#endpoint = EndPoint
bucket = oss2.Bucket(auth, EndPoint, Bucket)
oss2.resumable_download(bucket,
RemoteFile,
LocalFile,
store=oss2.ResumableDownloadStore(root='/tmp'),
multiget_threshold=20 * 1024 * 1024,
part_size=10 * 1024 * 1024,
num_threads=3)
def _get_samples_from_index(self, bucket):
logger.info(f"Index path {self._path_index()}")
with tempfile.NamedTemporaryFile() as file_tmp:
# Used intrad of put_object to avoid timeouts
oss2.resumable_download(bucket,
self._path_index(),
file_tmp.name,
num_threads=5)
with open(file_tmp.name, 'rb') as fp:
data = fp.read()
logger.info(f"Index size: {len(data)}")
self.keys_relative, self.classes, self.class_to_idx = pickle.loads(
zlib.decompress(data))
# Patch for indices including ''
if '' in self.classes:
self.classes, self.class_to_idx = self._find_classes()
def test_resumable_incomplete_download(self):
"""One of the part is incomplete, while there's no exception raised."""
oss2.defaults.multiget_threshold = 1
oss2.defaults.multiget_part_size = 100
oss2.defaults.multiget_num_threads = 5
file_size = 123 * 3 + 1
key, filename, content = self.__prepare(file_size)
with patch.object(oss2.Bucket, 'get_object',
side_effect=partial(mock_get_object, content_length=file_size),
autospec=True):
try:
oss2.resumable_download(self.bucket, key, filename)
except oss2.exceptions.InconsistentError as e:
self.assertTrue(e.request_id)
self.assertEqual(e.body, 'InconsistentError: IncompleteRead from source')
except:
self.assertTrue(False)
def test_resumable_incomplete_download(self):
"""One of the part is incomplete, while there's no exception raised."""
oss2.defaults.multiget_threshold = 1
oss2.defaults.multiget_part_size = 100
oss2.defaults.multiget_num_threads = 5
file_size = 123 * 3 + 1
key, filename, content = self.__prepare(file_size)
with patch.object(oss2.Bucket, 'get_object',
side_effect=partial(mock_get_object, content_length=file_size),
autospec=True):
try:
oss2.resumable_download(self.bucket, key, filename)
except oss2.exceptions.InconsistentError as e:
self.assertTrue(e.request_id)
self.assertEqual(e.body, 'InconsistentError: IncompleteRead from source')
except:
self.assertTrue(False)
def test_progress(self):
oss2.defaults.multiget_threshold = 1
oss2.defaults.multiget_part_size = 100
oss2.defaults.multiget_num_threads = 1
stats = {'previous': -1, 'called':0}
def progress_callback(bytes_consumed, total_bytes):
self.assertTrue(bytes_consumed <= total_bytes)
self.assertTrue(bytes_consumed > stats['previous'])
stats['previous'] = bytes_consumed
stats['called'] += 1
file_size = 100 * 5 + 1
key, filename, content = self.__prepare(file_size)
oss2.resumable_download(self.bucket, key, filename, progress_callback=progress_callback)
self.assertEqual(stats['previous'], file_size)
self.assertEqual(stats['called'], oss2.utils.how_many(file_size, oss2.defaults.multiget_part_size) + 1)
def get_file(self, rpath, lpath, callback=None, **kwargs): # pylint: disable=arguments-differ
"""
Copy single remote file to local
"""
kwargs.setdefault("progress_callback", _as_progress_handler(callback))
if self.isdir(rpath):
os.makedirs(lpath, exist_ok=True)
else:
bucket_name, obj_name = self.split_path(rpath)
connect_timeout = kwargs.pop("connect_timeout", None)
bucket = self._get_bucket(bucket_name, connect_timeout)
if self.size(rpath) >= self.SIMPLE_TRANSFER_THRESHOLD:
oss2.resumable_download(bucket, obj_name, lpath, **kwargs)
else:
self._call_oss(
"get_object_to_file",
obj_name,
lpath,
bucket=bucket_name,
timeout=connect_timeout,
**kwargs,
)
def test_resumable_down1(self):
key = 'requestpayment-test-resumable-down-object'
content = b'a' * (2 * 1024 * 1024)
file_name = key + '.txt'
self.bucket.put_object(key, content)
# Resumale down object smaller than multiget_threshold without payer setting, should be failed.
self.assertRaises(oss2.exceptions.ServerError, oss2.resumable_download, self.payer_bucket, key, file_name,
multiget_threshold=(3*1024*1024), num_threads=2, part_size=(100*1024))
# Resumale down object smaller than multiget_threshold with payer setting, should be successful.
headers = dict()
headers[OSS_REQUEST_PAYER] = "requester"
oss2.resumable_download(self.payer_bucket, key, file_name,
multiget_threshold=(3*1024*1024), num_threads=2, part_size=(100*1024), headers=headers)
# Check file size
file_size = os.stat(file_name).st_size
self.assertEqual(file_size, (2*1024*1024))
os.remove(file_name)
# Resumale down object bigger than multiget_threshold without payer setting, should be failed.
self.assertRaises(oss2.exceptions.ServerError, oss2.resumable_download, self.payer_bucket, key, file_name,
multiget_threshold=(500*1024), num_threads=2, part_size=(100*1024))
# Resumale down object bigger than multiget_threshold with payer setting, should be successful.
headers = dict()
headers[OSS_REQUEST_PAYER] = "requester"
oss2.resumable_download(self.payer_bucket, key, file_name,
multiget_threshold=(500*1024), num_threads=2, part_size=(100*1024), headers=headers)
# Check file size
file_size = os.stat(file_name).st_size
self.assertEqual(file_size, (2*1024*1024))
os.remove(file_name)
self.bucket.delete_object(key)
def __test_insane_record(self, file_size, modify_record_func, old_tmp_exists=True):
orig_rename = os.rename
obj = NonlocalObject({})
key, filename, content = self.__prepare(file_size)
def mock_rename(src, dst):
obj.var = self.__record(key, filename)
orig_rename(src, dst)
with patch.object(os, 'rename', side_effect=RuntimeError(), autospec=True):
self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename)
store = oss2.resumable.make_download_store()
store_key = store.make_store_key(self.bucket.bucket_name, key, os.path.abspath(filename))
r = store.get(store_key)
modify_record_func(store, store_key, copy.deepcopy(r))
with patch.object(os, 'rename', side_effect=mock_rename, autospec=True):
oss2.resumable_download(self.bucket, key, filename)
new_r = obj.var
self.assertTrue(new_r['tmp_suffix'] != r['tmp_suffix'])
self.assertEqual(new_r['size'], r['size'])
self.assertEqual(new_r['mtime'], r['mtime'])
self.assertEqual(new_r['etag'], r['etag'])
self.assertEqual(new_r['part_size'], r['part_size'])
self.assertEqual(os.path.exists(filename + r['tmp_suffix']), old_tmp_exists)
self.assertTrue(not os.path.exists(filename + new_r['tmp_suffix']))
oss2.utils.silently_remove(filename + r['tmp_suffix'])
result = bucket.get_object(key, progress_callback=percentage)
content_got = b''
for chunk in result:
content_got += chunk
assert content == content_got
"""
范围下载
"""
# 带进度条的范围下载
result = bucket.get_object(key, byte_range=(1024, 2047), progress_callback=percentage)
content_got = b''
for chunk in result:
content_got += chunk
assert 'a'*1024 == content_got
"""
断点续传下载
"""
# 带进度条的断点续传下载
filename = 'download.txt'
oss2.resumable_download(bucket, key, filename,
multiget_threshold=200*1024,
part_size=100*1024,
num_threads=3,
progress_callback=percentage)
os.remove(filename)
# 删除上传的文件
bucket.delete_object(key)
def test_relpath_and_abspath(self):
# """测试绝对、相对路径"""
# testing steps:
# 1. first use abspath, and fail one part
# 2. then use relpath to continue
bucket = random.choice(
[self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket])
cwd = os.getcwd()
if os.name == 'nt':
os.chdir('C:\\')
oss2.defaults.multiget_threshold = 1
oss2.defaults.multiget_part_size = 100
oss2.defaults.multiget_num_threads = 5
oss2.defaults.min_part_size = 100
fd, abspath = tempfile.mkstemp()
os.close(fd)
relpath = os.path.relpath(abspath)
self.assertNotEqual(abspath, relpath)
file_size = 1000
key = self.random_key()
content = random_bytes(file_size)
bucket.put_object(key, content)
orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part
orig_rename = os.rename
context1 = {}
context2 = {}
def mock_download_part(downloader, part, part_number=None):
if part.part_number == part_number:
r = self.__record(key, abspath)
context1['file_path'] = r['file_path']
context1['tmp_suffix'] = r['tmp_suffix']
raise RuntimeError(
"Fail download_part for part: {0}".format(part_number))
else:
orig_download_part(downloader, part)
def mock_rename(src, dst):
r = self.__record(key, relpath)
context2['file_path'] = r['file_path']
context2['tmp_suffix'] = r['tmp_suffix']
orig_rename(src, dst)
with patch.object(oss2.resumable._ResumableDownloader,
'_ResumableDownloader__download_part',
side_effect=partial(mock_download_part,
part_number=3),
autospec=True):
self.assertRaises(RuntimeError, oss2.resumable_download, bucket,
key, abspath)
with patch.object(os, 'rename', side_effect=mock_rename):
oss2.resumable_download(bucket, key, relpath)
self.assertEqual(context1['file_path'], context2['file_path'])
self.assertEqual(context1['tmp_suffix'], context2['tmp_suffix'])
oss2.utils.silently_remove(abspath)
if os.name == 'nt':
os.chdir(cwd)
def test_relpath_and_abspath(self):
"""测试绝对、相对路径"""
# testing steps:
# 1. first use abspath, and fail one part
# 2. then use relpath to continue
if os.name == 'nt':
os.chdir('C:\\')
oss2.defaults.multiget_threshold = 1
oss2.defaults.multiget_part_size = 100
oss2.defaults.multiget_num_threads = 5
fd, abspath = tempfile.mkstemp()
os.close(fd)
relpath = os.path.relpath(abspath)
self.assertNotEqual(abspath, relpath)
file_size = 1000
key = self.random_key()
content = random_bytes(file_size)
self.bucket.put_object(key, content)
orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part
orig_rename = os.rename
context1 = {}
context2 = {}
def mock_download_part(downloader, part, part_number=None):
if part.part_number == part_number:
r = self.__record(key, abspath)
context1['abspath'] = r['abspath']
context1['tmp_suffix'] = r['tmp_suffix']
raise RuntimeError("Fail download_part for part: {0}".format(part_number))
else:
orig_download_part(downloader, part)
def mock_rename(src, dst):
r = self.__record(key, relpath)
context2['abspath'] = r['abspath']
context2['tmp_suffix'] = r['tmp_suffix']
orig_rename(src, dst)
with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part',
side_effect=partial(mock_download_part, part_number=3),
autospec=True):
self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, abspath)
with patch.object(os, 'rename', side_effect=mock_rename):
oss2.resumable_download(self.bucket, key, relpath)
self.assertEqual(context1['abspath'], context2['abspath'])
self.assertEqual(context1['tmp_suffix'], context2['tmp_suffix'])
oss2.utils.silently_remove(abspath)
content_got = b''
for chunk in result:
content_got += chunk
assert content_got == 'a'*1024
# 范围下载到本地文件
result = bucket.get_object_to_file(key, filename, byte_range=(1024, 2047))
# 验证一下
with open(filename, 'rb') as fileobj:
assert fileobj.read() == 'a'*1024
"""
断点续传下载
"""
# 断点续传下载
oss2.resumable_download(bucket, key, filename,
multiget_threshold=200*1024,
part_size=100*1024,
num_threads=3)
# 验证一下
with open(filename, 'rb') as fileobj:
assert fileobj.read() == content
# 清理文件
os.remove(filename)
def __test_crypto_normal(self, file_size):
key, filename, content = self.__prepare(file_size, useCrypto=True)
oss2.resumable_download(self.rsa_crypto_bucket, key, filename)
self.assertFileContent(filename, content)
def __test_normal(self, file_size):
key, filename, content = self.__prepare(file_size)
oss2.resumable_download(self.bucket, key, filename)
self.assertFileContent(filename, content)
