def __test_normal(self, file_size): bucket = random.choice( [self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket]) key, filename, content = self.__prepare(bucket, file_size) oss2.resumable_download(bucket, key, filename) self.assertFileContent(filename, content)
def downloadFiles(bucket): """ downloadFiles download FLAGS.files on the oss """ if not os.path.exists(FLAGS.outputPath): os.makedirs(FLAGS.outputPath) print("The floder {0} is not existed, will creat it".format( FLAGS.outputPath)) start_time = time.time() for tmp_file in FLAGS.files: if not bucket.object_exists(tmp_file): print("File {0} is not on the OSS!".format(tmp_file)) else: print("Will download {0} !".format(tmp_file)) tmp_time = time.time() # cut the file name filename = tmp_file[tmp_file.rfind("/") + 1:len(tmp_file)] localFilename = os.path.join(FLAGS.outputPath, filename) # bucket.get_object_to_file( oss2.resumable_download(bucket, tmp_file, localFilename, progress_callback=percentage) print("\nFile {0} -> {1} downloads finished, cost {2} Sec.".format( tmp_file, localFilename, time.time() - tmp_time)) print("All download tasks have finished!") print("Cost {0} Sec.".format(time.time() - start_time))
def download_fonts(self, keyword='', pref=''): for fonts_bucket in iter(self.get_font_bucket()): # get bucket region fonts_bucket_info = fonts_bucket.get_bucket_info() region = fonts_bucket_info.location.split('-')[-1] # get font bucket sotorage dir fonts_dir = config.ali_fonts_bucket[region]['font_dir'] bucket_name = fonts_bucket.bucket_name # oss2.ObjectIteratorr用于遍历文件。 oss_object_list = oss2.ObjectIterator(fonts_bucket) for font_file in oss_object_list: file_name = font_file.key.split(fonts_dir)[-1] if file_name.endswith('tf') and keyword in file_name and file_name.startswith(pref) and fonts_dir in font_file.key: print('fonts %s matched for download in bucket %s' % (font_file.key, fonts_bucket.bucket_name)) self.logger.info( 'fonts %s matched for download in bucket %s' % (font_file.key, fonts_bucket.bucket_name)) try: oss2.resumable_download(fonts_bucket, font_file, '../downloads/' + font_file, part_size=100 * 1024, num_threads=3, progress_callback=self.percentage, store=oss2.ResumableDownloadStore(root='./tmp_files/downloads')) except oss2.exceptions.NotFound as en: self.logger.exception('Font %s not found while download fonts' % font_file) except Exception as e: self.logger.exception('Exception catched while download fonts %s: %s' % (font_file, e)) else: # print('fonts %s not matched for download in bucket %s' % (file_name, fonts_bucket.bucket_name)) self.logger.debug('fonts %s not matched for download in bucket %s' % (file_name, fonts_bucket.bucket_name))
def __test_resume(self, file_size, failed_parts, modify_func_record=None): total = NonlocalObject(0) orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part def mock_download_part(self, part, failed_parts=None): if part.part_number in failed_parts: raise RuntimeError("Fail download_part for part: {0}".format(part.part_number)) else: total.var += 1 orig_download_part(self, part) key, filename, content = self.__prepare(file_size) with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part', side_effect=partial(mock_download_part, failed_parts=failed_parts), autospec=True): self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename) store = oss2.resumable.make_download_store() store_key = store.make_store_key(self.bucket.bucket_name, key, os.path.abspath(filename)) record = store.get(store_key) tmp_file = filename + record['tmp_suffix'] self.assertTrue(os.path.exists(tmp_file)) self.assertTrue(not os.path.exists(filename)) with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part', side_effect=partial(mock_download_part, failed_parts=[]), autospec=True): oss2.resumable_download(self.bucket, key, filename) self.assertEqual(total.var, oss2.utils.how_many(file_size, oss2.defaults.multiget_part_size)) self.assertTrue(not os.path.exists(tmp_file)) self.assertFileContent(filename, content)
def downloadfile(self, rpath, lpath, **kwargs): """ :param rpath: 远程文件路径 :param lpath: 本地文件路径 :return: """ oss2.resumable_download(self.bucket, rpath, lpath, **kwargs)
def test_tmp_file_removed(self): oss2.defaults.multiget_threshold = 1 oss2.defaults.multiget_part_size = 100 oss2.defaults.multiget_num_threads = 5 orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part file_size = 123 * 3 + 1 key, filename, content = self.__prepare(file_size) context = {} def mock_download_part(downloader, part, part_number=None): if part.part_number == part_number: r = self.__record(key, filename) context['tmpfile'] = filename + r['tmp_suffix'] raise RuntimeError("Fail download_part for part: {0}".format(part_number)) else: orig_download_part(downloader, part) with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part', side_effect=partial(mock_download_part, part_number=2), autospec=True): self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename) os.remove(context['tmpfile']) oss2.resumable_download(self.bucket, key, filename) self.assertFileContent(filename, content)
def get_obj(self, key, dst, use_resume=True, part_size=(20 * 1024 * 1024), num_threads=4): """ get files from oss, :param key: oss key :param dst: The path to save obj. :return obj save path at last """ try: if self.is_public_net: raise Exception('Do not download from public') if use_resume: oss2.resumable_download( self.bucket, key, dst, store=oss2.ResumableDownloadStore(root='/tmp'), multiget_threshold=20 * 1024 * 1024, part_size=part_size, num_threads=num_threads) else: self.bucket.get_object_to_file(key, dst) return dst except Exception as ex: print ex.message return None
def test_tmp_file_removed(self): oss2.defaults.multiget_threshold = 1 oss2.defaults.multiget_part_size = 100 oss2.defaults.multiget_num_threads = 5 orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part file_size = 123 * 3 + 1 key, filename, content = self.__prepare(file_size) context = {} def mock_download_part(downloader, part, part_number=None): if part.part_number == part_number: r = self.__record(key, filename) context['tmpfile'] = filename + r['tmp_suffix'] raise RuntimeError( "Fail download_part for part: {0}".format(part_number)) else: orig_download_part(downloader, part) with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part', side_effect=partial(mock_download_part, part_number=2), autospec=True): self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename) os.remove(context['tmpfile']) oss2.resumable_download(self.bucket, key, filename) self.assertFileContent(filename, content)
def test_progress(self): bucket = random.choice( [self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket]) oss2.defaults.multiget_threshold = 1 oss2.defaults.multiget_part_size = 100 oss2.defaults.multiget_num_threads = 1 oss2.defaults.min_part_size = 100 stats = {'previous': -1, 'called': 0} def progress_callback(bytes_consumed, total_bytes): self.assertTrue(bytes_consumed <= total_bytes) self.assertTrue(bytes_consumed > stats['previous']) stats['previous'] = bytes_consumed stats['called'] += 1 file_size = 100 * 5 + 1 key, filename, content = self.__prepare(bucket, file_size) oss2.resumable_download(bucket, key, filename, progress_callback=progress_callback) self.assertEqual(stats['previous'], file_size) self.assertEqual( stats['called'], oss2.utils.how_many(file_size, oss2.defaults.multiget_part_size) + 1)
def download(self, ossObject, loaclFile): oss2.resumable_download(self.bucket, ossObject, loaclFile, store=oss2.ResumableDownloadStore(root=os.path.dirname(loaclFile)), multiget_threshold=1 * 1024, part_size=10 * 1024 * 1024, num_threads=3, progress_callback= self.percentage )
def download(file, local_name): """resume download orc file from oss""" logger.info(f'start download oss file {file}....') try: oss2.resumable_download(oss_bucket, file, local_name, num_threads=3) except Exception as e: logger.error(e) download(oss_bucket, local_name)
def downloadFile(self, path, localPath): oss2.resumable_download( self._bucket, path, localPath, store=oss2.ResumableDownloadStore(root='./tmp'), multiget_threshold=20 * 1024 * 1024, part_size=10 * 1024 * 1024, num_threads=3)
def test_resume_rename_failed(self): bucket = random.choice( [self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket]) size = 500 * 10 part_size = 499 oss2.defaults.multiget_threshold = 1 oss2.defaults.multiget_part_size = part_size oss2.defaults.multiget_num_threads = 3 oss2.defaults.min_part_size = 499 key, filename, content = self.__prepare(bucket, size) with patch.object(os, 'rename', side_effect=RuntimeError(), autospec=True): self.assertRaises(RuntimeError, oss2.resumable_download, bucket, key, filename) r = self.__record(key, filename) # assert record fields are valid head_object_result = bucket.head_object(key) self.assertEqual(r['size'], size) self.assertEqual(r['mtime'], head_object_result.last_modified) self.assertEqual(r['etag'], head_object_result.etag) self.assertEqual(r['bucket'], bucket.bucket_name) self.assertEqual(r['key'], key) self.assertEqual(r['part_size'], part_size) self.assertTrue(os.path.exists(filename + r['tmp_suffix'])) self.assertFileContent(filename + r['tmp_suffix'], content) self.assertTrue(not os.path.exists(filename)) self.assertEqual(r['file_path'], os.path.abspath(filename)) self.assertEqual(len(r['parts']), oss2.utils.how_many(size, part_size)) parts = sorted(r['parts'], key=lambda p: p['part_number']) for i, p in enumerate(parts): self.assertEqual(p['part_number'], i + 1) self.assertEqual(p['start'], part_size * i) self.assertEqual(p['end'], min(part_size * (i + 1), size)) with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part', side_effect=RuntimeError(), autospec=True): oss2.resumable_download(bucket, key, filename) self.assertTrue(not os.path.exists(filename + r['tmp_suffix'])) self.assertFileContent(filename, content) self.assertEqual(self.__record(key, filename), None)
def download(self,url): if bucket.object_exists(url): local_f = "/data1/jenkins_dir/" + url oss2.resumable_download(bucket, url, local_f, store=oss2.ResumableDownloadStore(root='/tmp'), multiget_threshold=20*1024*1024, part_size=10*1024*1024, num_threads=2) return True else: return False
def _download( self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs ): with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar: import oss2 oss2.resumable_download( self.oss_service, from_info.path, to_file, progress_callback=pbar.update_to, )
def test_remote_changed_during_download(self): bucket = random.choice( [self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket]) oss2.defaults.multiget_threshold = 1 oss2.defaults.multiget_part_size = 100 oss2.defaults.multiget_num_threads = 2 oss2.defaults.min_part_size = 100 orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part orig_rename = os.rename file_size = 1000 key, filename, content = self.__prepare(bucket, file_size) old_context = {} new_context = {} # 模拟download某个part的过程中原始文件的内容被修改了 def mock_download_part(downloader, part, part_number=None): if part.part_number == part_number: r = self.__record(key, filename) old_context['tmp_suffix'] = r['tmp_suffix'] old_context['etag'] = r['etag'] old_context['content'] = random_bytes(file_size) bucket.put_object(key, old_context['content']) orig_download_part(downloader, part) # rename之前将新的record获取到 def mock_rename(src, dst): r = self.__record(key, filename) new_context['tmp_suffix'] = r['tmp_suffix'] new_context['etag'] = r['etag'] orig_rename(src, dst) with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part', side_effect=partial(mock_download_part, part_number=5), autospec=True): self.assertRaises(oss2.exceptions.PreconditionFailed, oss2.resumable_download, bucket, key, filename) with patch.object(os, 'rename', side_effect=mock_rename): oss2.resumable_download(bucket, key, filename) self.assertTrue(new_context['tmp_suffix'] != old_context['tmp_suffix']) self.assertTrue(new_context['etag'] != old_context['etag'])
def test_two_downloaders(self): """两个downloader同时跑,但是store的目录不一样。""" oss2.defaults.multiget_threshold = 1 oss2.defaults.multiget_part_size = 100 oss2.defaults.multiget_num_threads = 2 store1 = oss2.make_download_store() store2 = oss2.make_download_store(dir='.another-py-oss-download') file_size = 1000 key, filename, content = self.__prepare(file_size) context1a = {} context1b = {} context2 = {} def mock_rename(src, dst, ctx=None, store=None): r = self.__record(key, filename, store=store) ctx['tmp_suffix'] = r['tmp_suffix'] ctx['etag'] = r['etag'] ctx['mtime'] = r['mtime'] raise RuntimeError('intentional') with patch.object(os, 'rename', side_effect=partial(mock_rename, ctx=context1a, store=store1), autospect=True): self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename, store=store1) with patch.object(os, 'rename', side_effect=partial(mock_rename, ctx=context1b, store=store1), autospect=True): self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename, store=store1) with patch.object(os, 'rename', side_effect=partial(mock_rename, ctx=context2, store=store2), autospect=True): self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename, store=store2) self.assertEqual(context1a['tmp_suffix'], context1b['tmp_suffix']) self.assertEqual(context1a['etag'], context1b['etag']) self.assertEqual(context1a['mtime'], context1b['mtime']) self.assertNotEqual(context1a['tmp_suffix'], context2['tmp_suffix']) self.assertEqual(context1a['etag'], context2['etag']) self.assertEqual(context1a['mtime'], context2['mtime']) self.assertTrue(os.path.exists(filename + context1a['tmp_suffix'])) self.assertTrue(os.path.exists(filename + context2['tmp_suffix'])) oss2.resumable_download(self.bucket, key, filename, store=store1) self.assertTrue(not os.path.exists(filename + context1a['tmp_suffix'])) self.assertTrue(os.path.exists(filename + context2['tmp_suffix'])) oss2.resumable_download(self.bucket, key, filename, store=store2) self.assertTrue(not os.path.exists(filename + context2['tmp_suffix']))
def test_resumable_download_with_version(self): from oss2.models import BucketVersioningConfig from oss2.models import BatchDeleteObjectVersion from oss2.models import BatchDeleteObjectVersionList auth = oss2.Auth(OSS_ID, OSS_SECRET) bucket_name = OSS_BUCKET + "-test-resumable-download-with-version" bucket = oss2.Bucket(auth, OSS_ENDPOINT, bucket_name) bucket.create_bucket(oss2.BUCKET_ACL_PRIVATE) wait_meta_sync() config = BucketVersioningConfig() config.status = 'Enabled' result = bucket.put_bucket_versioning(config) wait_meta_sync() self.version_bucket = bucket content_small = random_bytes(5 * 1024) result = bucket.put_object("object_small", content_small) version_small = result.versionid filename_small = self.random_filename() result = oss2.resumable_download(bucket, "object_small", filename_small) self.assertFileContent(filename_small, content_small) content_big = random_bytes(5 * 1024 * 1024) result = bucket.put_object("object_big", content_big) version_big = result.versionid filename_big = self.random_filename() result = oss2.resumable_download(bucket, "object_big", filename_big) self.assertFileContent(filename_big, content_big) version_list = BatchDeleteObjectVersionList() version_list.append( BatchDeleteObjectVersion("object_small", version_small)) version_list.append(BatchDeleteObjectVersion("object_big", version_big)) result = bucket.delete_object_versions(version_list) self.assertTrue(len(result.delete_versions) == 2) bucket.delete_bucket()
def test_resumable_download(self): key = 'traffic-limit-test-resumble-download-object' content = b'a' * OBJECT_SIZE_1MB file_name = key + '.txt' # Put object self.bucket.put_object(key, content) headers = dict() headers[OSS_TRAFFIC_LIMIT] = str(LIMIT_100KB) # Resumable download object smaller than multiget_threshold with traffic limit setting. start_time_sec = int(time.time()) oss2.resumable_download(self.bucket, key, file_name, multiget_threshold=(OBJECT_SIZE_1MB * 2), num_threads=1, headers=headers) self.assertFileContent(file_name, content) end_time_sec = int(time.time()) os.remove(file_name) # Calculate expensed time expense_time_sec = end_time_sec - start_time_sec # Theoretical time is 1MB/100KB = 10s, set the minimum theoretical time to 10*0.7s theoretical_exepnse_min = 10 * 0.7 # Compare to minimum theoretical time self.assertEqual((expense_time_sec > theoretical_exepnse_min), True) # Resumable download object bigger than multiget_threshold with traffic limit setting. start_time_sec = int(time.time()) oss2.resumable_download(self.bucket, key, file_name, multiget_threshold=(OBJECT_SIZE_1MB - 1024), num_threads=1, headers=headers) self.assertFileContent(file_name, content) end_time_sec = int(time.time()) os.remove(file_name) self.bucket.delete_object(key) # Calculate expensed time expense_time_sec = end_time_sec - start_time_sec # Theoretical time is 1MB/100KB = 10s, set the minimum theoretical time to 10*0.7s theoretical_exepnse_min = 10 * 0.7 # Compare to minimum theoretical time self.assertEqual((expense_time_sec > theoretical_exepnse_min), True)
def __test_insane_record(self, file_size, modify_record_func, old_tmp_exists=True): bucket = random.choice( [self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket]) orig_rename = os.rename obj = NonlocalObject({}) key, filename, content = self.__prepare(bucket, file_size) def mock_rename(src, dst): obj.var = self.__record(key, filename) orig_rename(src, dst) # 这个地方模拟rename报错 with patch.object(os, 'rename', side_effect=RuntimeError(), autospec=True): self.assertRaises(RuntimeError, oss2.resumable_download, bucket, key, filename) store = oss2.resumable.make_download_store() store_key = store.make_store_key(bucket.bucket_name, key, os.path.abspath(filename)) r = store.get(store_key) # 修改record文件 modify_record_func(store, store_key, copy.deepcopy(r)) with patch.object(os, 'rename', side_effect=mock_rename, autospec=True): oss2.resumable_download(bucket, key, filename) new_r = obj.var self.assertTrue(new_r['tmp_suffix'] != r['tmp_suffix']) self.assertEqual(new_r['size'], r['size']) self.assertEqual(new_r['mtime'], r['mtime']) self.assertEqual(new_r['etag'], r['etag']) self.assertEqual(new_r['part_size'], r['part_size']) self.assertEqual(os.path.exists(filename + r['tmp_suffix']), old_tmp_exists) self.assertTrue(not os.path.exists(filename + new_r['tmp_suffix'])) oss2.utils.silently_remove(filename + r['tmp_suffix'])
def test_download_success(test_bucket_access): import os import oss2 bucket = test_bucket_access key = "test_only.txt" filename = 'test_download_success.txt' oss2.resumable_download(bucket, key, filename, multiget_threshold=200 * 1024, part_size=100 * 1024, num_threads=3) assert filename in os.listdir() os.remove(filename)
def test_resume_rename_failed(self): size = 500 * 10 part_size = 499 oss2.defaults.multiget_threshold = 1 oss2.defaults.multiget_part_size = part_size oss2.defaults.multiget_num_threads = 3 key, filename, content = self.__prepare(size) with patch.object(os, 'rename', side_effect=RuntimeError(), autospec=True): self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename) r = self.__record(key, filename) # assert record fields are valid head_object_result = self.bucket.head_object(key) self.assertEqual(r['size'], size) self.assertEqual(r['mtime'], head_object_result.last_modified) self.assertEqual(r['etag'], head_object_result.etag) self.assertEqual(r['bucket'], self.bucket.bucket_name) self.assertEqual(r['key'], key) self.assertEqual(r['part_size'], part_size) self.assertTrue(os.path.exists(filename + r['tmp_suffix'])) self.assertFileContent(filename + r['tmp_suffix'], content) self.assertTrue(not os.path.exists(filename)) self.assertEqual(r['abspath'], os.path.abspath(filename)) self.assertEqual(len(r['parts']), oss2.utils.how_many(size, part_size)) parts = sorted(r['parts'], key=lambda p: p['part_number']) for i, p in enumerate(parts): self.assertEqual(p['part_number'], i+1) self.assertEqual(p['start'], part_size * i) self.assertEqual(p['end'], min(part_size*(i+1), size)) with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part', side_effect=RuntimeError(), autospec=True): oss2.resumable_download(self.bucket, key, filename) self.assertTrue(not os.path.exists(filename + r['tmp_suffix'])) self.assertFileContent(filename, content) self.assertEqual(self.__record(key, filename), None)
def download_from_oss(local_path, endpoint, remote_path): bucket = get_bucket(endpoint) print("Downloading {0}...".format(remote_path)) try: return resumable_download(bucket, remote_path, local_path) except (NotFound, NoSuchKey): # FIXME: 最多只能下载 1000 个文件 remote_files = _list_files(bucket, remote_path, MAX_FILE_COUNT) for remote_file in remote_files: rel = os.path.relpath(remote_file, remote_path) local_file = os.path.join(local_path, rel) local_file_dir = os.path.dirname(local_file) os.makedirs(local_file_dir, exist_ok=True) print("Downloading {0}...".format(remote_file)) resumable_download(bucket, remote_file, local_file)
def download(self, remote_file, local_file): ''' :param remote_file: 仓库文件名 :param local_file: 本地文件名 :return: ''' try: oss2.resumable_download(self.bucket, remote_file, local_file, multiget_threshold=200 * 1024, part_size=100 * 1024, num_threads=3, progress_callback=self.percentage) except oss2.exceptions.NotFound: print("下载的文件不存在!")
def test_resumable_down(self): small_object = 'requestpayment-test-resumable-down-small-object' content1 = b'a' * (150 * 1024) big_object = 'requestpayment-test-resumable-down-big-object' content2 = b'a' * (500 * 1024) file_name = small_object + '.txt' self.bucket.put_object(small_object, content1) self.bucket.put_object(big_object, content2) # Resumale down small object without payer setting, should be failed. self.assertRaises(oss2.exceptions.ServerError, oss2.resumable_download, self.payer_bucket, small_object, file_name, multiget_threshold=(200*1024), num_threads=2, part_size=(1024*1024)) # Resumale down small object with payer setting, should be successful. headers = dict() headers[OSS_REQUEST_PAYER] = "requester" oss2.resumable_download(self.payer_bucket, small_object, file_name, multiget_threshold=(200*1024), num_threads=2, part_size=(1024*1024), headers=headers) # Check file size file_size = os.stat(file_name).st_size self.assertEqual(file_size, (150*1024)) os.remove(file_name) self.bucket.delete_object(small_object) file_name = big_object + '.txt' # Resumale down big object without payer setting, should be failed. self.assertRaises(oss2.exceptions.ServerError, oss2.resumable_download, self.payer_bucket, big_object, file_name, multiget_threshold=(200*1024), num_threads=2, part_size=(1024*1024)) # Resumale down big object with payer setting, should be successful. headers = dict() headers[OSS_REQUEST_PAYER] = "requester" oss2.resumable_download(self.payer_bucket, big_object, file_name, multiget_threshold=(200*1024), num_threads=2, part_size=(1024*1024), headers=headers) # Check file size file_size = os.stat(file_name).st_size self.assertEqual(file_size, (500*1024)) os.remove(file_name) self.bucket.delete_object(big_object)
def download_war(dm_name, version, redis_key): #下载对应项目的最新war包 try: #包名需要规范 Redis.lpush( redis_key, '%s-%s.war package download from oss ......' % (dm_name, version)) auth = oss2.Auth(oss_id, oss_key) bucket = oss2.Bucket(auth, 'oss-cn-beijing.aliyuncs.com', 'mojiops') file_war = None for obj in oss2.ObjectIterator(bucket, prefix='tomcat-'): if '%s-%s' % (dm_name, version) in obj.key: file_war = obj.key break if not os.path.exists('%s/%s' % (dockerfile_path, dm_name)): os.mkdir('%s/%s' % (dockerfile_path, dm_name)) if file_war: object = file_war.split('/')[-1] project_path = '%s/%s/%s.war' % (dockerfile_path, dm_name, dm_name.split('-')[0]) if os.path.exists(project_path): os.remove(project_path) #尝试3次下载 for i in range(3): try: oss2.resumable_download(bucket, file_war, project_path) except Exception as e: logging.error(e) else: if os.path.exists(project_path): #生成dockerfile文件 # Redis.lpush(redis_key, '%s package download success!' % object) return object else: Redis.lpush(redis_key, '%s package not fond!' % object) return None if not os.path.exists(project_path): Redis.lpush(redis_key, '%s package download fail!' % object) return None except Exception as e: logging.error(e) Redis.lpush(redis_key, '%s package download fail!' % object) return None
def test_remote_changed_during_download(self): oss2.defaults.multiget_threshold = 1 oss2.defaults.multiget_part_size = 100 oss2.defaults.multiget_num_threads = 2 orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part orig_rename = os.rename file_size = 1000 key, filename, content = self.__prepare(file_size) old_context = {} new_context = {} def mock_download_part(downloader, part, part_number=None): if part.part_number == part_number: r = self.__record(key, filename) old_context['tmp_suffix'] = r['tmp_suffix'] old_context['etag'] = r['etag'] old_context['content'] = random_bytes(file_size) self.bucket.put_object(key, old_context['content']) orig_download_part(downloader, part) def mock_rename(src, dst): r = self.__record(key, filename) new_context['tmp_suffix'] = r['tmp_suffix'] new_context['etag'] = r['etag'] orig_rename(src, dst) with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part', side_effect=partial(mock_download_part, part_number=5), autospec=True): self.assertRaises(oss2.exceptions.PreconditionFailed, oss2.resumable_download, self.bucket, key, filename) with patch.object(os, 'rename', side_effect=mock_rename): oss2.resumable_download(self.bucket, key, filename) self.assertTrue(new_context['tmp_suffix'] != old_context['tmp_suffix']) self.assertTrue(new_context['etag'] != old_context['etag'])
def OssDownload(): accessKeyId = sys.argv[1] accessSecret = sys.argv[2] EndPoint = sys.argv[3] Bucket = sys.argv[4] RemoteFile = sys.argv[5] LocalFile = sys.argv[6] auth = oss2.Auth(accessKeyId, accessSecret) #endpoint = EndPoint bucket = oss2.Bucket(auth, EndPoint, Bucket) oss2.resumable_download(bucket, RemoteFile, LocalFile, store=oss2.ResumableDownloadStore(root='/tmp'), multiget_threshold=20 * 1024 * 1024, part_size=10 * 1024 * 1024, num_threads=3)
def _get_samples_from_index(self, bucket): logger.info(f"Index path {self._path_index()}") with tempfile.NamedTemporaryFile() as file_tmp: # Used intrad of put_object to avoid timeouts oss2.resumable_download(bucket, self._path_index(), file_tmp.name, num_threads=5) with open(file_tmp.name, 'rb') as fp: data = fp.read() logger.info(f"Index size: {len(data)}") self.keys_relative, self.classes, self.class_to_idx = pickle.loads( zlib.decompress(data)) # Patch for indices including '' if '' in self.classes: self.classes, self.class_to_idx = self._find_classes()
def test_resumable_incomplete_download(self): """One of the part is incomplete, while there's no exception raised.""" oss2.defaults.multiget_threshold = 1 oss2.defaults.multiget_part_size = 100 oss2.defaults.multiget_num_threads = 5 file_size = 123 * 3 + 1 key, filename, content = self.__prepare(file_size) with patch.object(oss2.Bucket, 'get_object', side_effect=partial(mock_get_object, content_length=file_size), autospec=True): try: oss2.resumable_download(self.bucket, key, filename) except oss2.exceptions.InconsistentError as e: self.assertTrue(e.request_id) self.assertEqual(e.body, 'InconsistentError: IncompleteRead from source') except: self.assertTrue(False)
def test_progress(self): oss2.defaults.multiget_threshold = 1 oss2.defaults.multiget_part_size = 100 oss2.defaults.multiget_num_threads = 1 stats = {'previous': -1, 'called':0} def progress_callback(bytes_consumed, total_bytes): self.assertTrue(bytes_consumed <= total_bytes) self.assertTrue(bytes_consumed > stats['previous']) stats['previous'] = bytes_consumed stats['called'] += 1 file_size = 100 * 5 + 1 key, filename, content = self.__prepare(file_size) oss2.resumable_download(self.bucket, key, filename, progress_callback=progress_callback) self.assertEqual(stats['previous'], file_size) self.assertEqual(stats['called'], oss2.utils.how_many(file_size, oss2.defaults.multiget_part_size) + 1)
def get_file(self, rpath, lpath, callback=None, **kwargs): # pylint: disable=arguments-differ """ Copy single remote file to local """ kwargs.setdefault("progress_callback", _as_progress_handler(callback)) if self.isdir(rpath): os.makedirs(lpath, exist_ok=True) else: bucket_name, obj_name = self.split_path(rpath) connect_timeout = kwargs.pop("connect_timeout", None) bucket = self._get_bucket(bucket_name, connect_timeout) if self.size(rpath) >= self.SIMPLE_TRANSFER_THRESHOLD: oss2.resumable_download(bucket, obj_name, lpath, **kwargs) else: self._call_oss( "get_object_to_file", obj_name, lpath, bucket=bucket_name, timeout=connect_timeout, **kwargs, )
def test_resumable_down1(self): key = 'requestpayment-test-resumable-down-object' content = b'a' * (2 * 1024 * 1024) file_name = key + '.txt' self.bucket.put_object(key, content) # Resumale down object smaller than multiget_threshold without payer setting, should be failed. self.assertRaises(oss2.exceptions.ServerError, oss2.resumable_download, self.payer_bucket, key, file_name, multiget_threshold=(3*1024*1024), num_threads=2, part_size=(100*1024)) # Resumale down object smaller than multiget_threshold with payer setting, should be successful. headers = dict() headers[OSS_REQUEST_PAYER] = "requester" oss2.resumable_download(self.payer_bucket, key, file_name, multiget_threshold=(3*1024*1024), num_threads=2, part_size=(100*1024), headers=headers) # Check file size file_size = os.stat(file_name).st_size self.assertEqual(file_size, (2*1024*1024)) os.remove(file_name) # Resumale down object bigger than multiget_threshold without payer setting, should be failed. self.assertRaises(oss2.exceptions.ServerError, oss2.resumable_download, self.payer_bucket, key, file_name, multiget_threshold=(500*1024), num_threads=2, part_size=(100*1024)) # Resumale down object bigger than multiget_threshold with payer setting, should be successful. headers = dict() headers[OSS_REQUEST_PAYER] = "requester" oss2.resumable_download(self.payer_bucket, key, file_name, multiget_threshold=(500*1024), num_threads=2, part_size=(100*1024), headers=headers) # Check file size file_size = os.stat(file_name).st_size self.assertEqual(file_size, (2*1024*1024)) os.remove(file_name) self.bucket.delete_object(key)
def __test_insane_record(self, file_size, modify_record_func, old_tmp_exists=True): orig_rename = os.rename obj = NonlocalObject({}) key, filename, content = self.__prepare(file_size) def mock_rename(src, dst): obj.var = self.__record(key, filename) orig_rename(src, dst) with patch.object(os, 'rename', side_effect=RuntimeError(), autospec=True): self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, filename) store = oss2.resumable.make_download_store() store_key = store.make_store_key(self.bucket.bucket_name, key, os.path.abspath(filename)) r = store.get(store_key) modify_record_func(store, store_key, copy.deepcopy(r)) with patch.object(os, 'rename', side_effect=mock_rename, autospec=True): oss2.resumable_download(self.bucket, key, filename) new_r = obj.var self.assertTrue(new_r['tmp_suffix'] != r['tmp_suffix']) self.assertEqual(new_r['size'], r['size']) self.assertEqual(new_r['mtime'], r['mtime']) self.assertEqual(new_r['etag'], r['etag']) self.assertEqual(new_r['part_size'], r['part_size']) self.assertEqual(os.path.exists(filename + r['tmp_suffix']), old_tmp_exists) self.assertTrue(not os.path.exists(filename + new_r['tmp_suffix'])) oss2.utils.silently_remove(filename + r['tmp_suffix'])
result = bucket.get_object(key, progress_callback=percentage) content_got = b'' for chunk in result: content_got += chunk assert content == content_got """ 范围下载 """ # 带进度条的范围下载 result = bucket.get_object(key, byte_range=(1024, 2047), progress_callback=percentage) content_got = b'' for chunk in result: content_got += chunk assert 'a'*1024 == content_got """ 断点续传下载 """ # 带进度条的断点续传下载 filename = 'download.txt' oss2.resumable_download(bucket, key, filename, multiget_threshold=200*1024, part_size=100*1024, num_threads=3, progress_callback=percentage) os.remove(filename) # 删除上传的文件 bucket.delete_object(key)
def test_relpath_and_abspath(self): # """测试绝对、相对路径""" # testing steps: # 1. first use abspath, and fail one part # 2. then use relpath to continue bucket = random.choice( [self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket]) cwd = os.getcwd() if os.name == 'nt': os.chdir('C:\\') oss2.defaults.multiget_threshold = 1 oss2.defaults.multiget_part_size = 100 oss2.defaults.multiget_num_threads = 5 oss2.defaults.min_part_size = 100 fd, abspath = tempfile.mkstemp() os.close(fd) relpath = os.path.relpath(abspath) self.assertNotEqual(abspath, relpath) file_size = 1000 key = self.random_key() content = random_bytes(file_size) bucket.put_object(key, content) orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part orig_rename = os.rename context1 = {} context2 = {} def mock_download_part(downloader, part, part_number=None): if part.part_number == part_number: r = self.__record(key, abspath) context1['file_path'] = r['file_path'] context1['tmp_suffix'] = r['tmp_suffix'] raise RuntimeError( "Fail download_part for part: {0}".format(part_number)) else: orig_download_part(downloader, part) def mock_rename(src, dst): r = self.__record(key, relpath) context2['file_path'] = r['file_path'] context2['tmp_suffix'] = r['tmp_suffix'] orig_rename(src, dst) with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part', side_effect=partial(mock_download_part, part_number=3), autospec=True): self.assertRaises(RuntimeError, oss2.resumable_download, bucket, key, abspath) with patch.object(os, 'rename', side_effect=mock_rename): oss2.resumable_download(bucket, key, relpath) self.assertEqual(context1['file_path'], context2['file_path']) self.assertEqual(context1['tmp_suffix'], context2['tmp_suffix']) oss2.utils.silently_remove(abspath) if os.name == 'nt': os.chdir(cwd)
def test_relpath_and_abspath(self): """测试绝对、相对路径""" # testing steps: # 1. first use abspath, and fail one part # 2. then use relpath to continue if os.name == 'nt': os.chdir('C:\\') oss2.defaults.multiget_threshold = 1 oss2.defaults.multiget_part_size = 100 oss2.defaults.multiget_num_threads = 5 fd, abspath = tempfile.mkstemp() os.close(fd) relpath = os.path.relpath(abspath) self.assertNotEqual(abspath, relpath) file_size = 1000 key = self.random_key() content = random_bytes(file_size) self.bucket.put_object(key, content) orig_download_part = oss2.resumable._ResumableDownloader._ResumableDownloader__download_part orig_rename = os.rename context1 = {} context2 = {} def mock_download_part(downloader, part, part_number=None): if part.part_number == part_number: r = self.__record(key, abspath) context1['abspath'] = r['abspath'] context1['tmp_suffix'] = r['tmp_suffix'] raise RuntimeError("Fail download_part for part: {0}".format(part_number)) else: orig_download_part(downloader, part) def mock_rename(src, dst): r = self.__record(key, relpath) context2['abspath'] = r['abspath'] context2['tmp_suffix'] = r['tmp_suffix'] orig_rename(src, dst) with patch.object(oss2.resumable._ResumableDownloader, '_ResumableDownloader__download_part', side_effect=partial(mock_download_part, part_number=3), autospec=True): self.assertRaises(RuntimeError, oss2.resumable_download, self.bucket, key, abspath) with patch.object(os, 'rename', side_effect=mock_rename): oss2.resumable_download(self.bucket, key, relpath) self.assertEqual(context1['abspath'], context2['abspath']) self.assertEqual(context1['tmp_suffix'], context2['tmp_suffix']) oss2.utils.silently_remove(abspath)
content_got = b'' for chunk in result: content_got += chunk assert content_got == 'a'*1024 # 范围下载到本地文件 result = bucket.get_object_to_file(key, filename, byte_range=(1024, 2047)) # 验证一下 with open(filename, 'rb') as fileobj: assert fileobj.read() == 'a'*1024 """ 断点续传下载 """ # 断点续传下载 oss2.resumable_download(bucket, key, filename, multiget_threshold=200*1024, part_size=100*1024, num_threads=3) # 验证一下 with open(filename, 'rb') as fileobj: assert fileobj.read() == content # 清理文件 os.remove(filename)
def __test_crypto_normal(self, file_size): key, filename, content = self.__prepare(file_size, useCrypto=True) oss2.resumable_download(self.rsa_crypto_bucket, key, filename) self.assertFileContent(filename, content)
def __test_normal(self, file_size): key, filename, content = self.__prepare(file_size) oss2.resumable_download(self.bucket, key, filename) self.assertFileContent(filename, content)