def download_fonts(self, keyword='', pref=''):
for fonts_bucket in iter(self.get_font_bucket()):
# get bucket region
fonts_bucket_info = fonts_bucket.get_bucket_info()
region = fonts_bucket_info.location.split('-')[-1]
# get font bucket sotorage dir
fonts_dir = config.ali_fonts_bucket[region]['font_dir']
bucket_name = fonts_bucket.bucket_name
# oss2.ObjectIteratorr用于遍历文件。
oss_object_list = oss2.ObjectIterator(fonts_bucket)
for font_file in oss_object_list:
file_name = font_file.key.split(fonts_dir)[-1]
if file_name.endswith('tf') and keyword in file_name and file_name.startswith(pref) and fonts_dir in font_file.key:
print('fonts %s matched for download in bucket %s' % (font_file.key, fonts_bucket.bucket_name))
self.logger.info(
'fonts %s matched for download in bucket %s' % (font_file.key, fonts_bucket.bucket_name))
try:
oss2.resumable_download(fonts_bucket, font_file, '../downloads/' + font_file,
part_size=100 * 1024,
num_threads=3,
progress_callback=self.percentage,
store=oss2.ResumableDownloadStore(root='./tmp_files/downloads'))
except oss2.exceptions.NotFound as en:
self.logger.exception('Font %s not found while download fonts' % font_file)
except Exception as e:
self.logger.exception('Exception catched while download fonts %s: %s' % (font_file, e))
else:
# print('fonts %s not matched for download in bucket %s' % (file_name, fonts_bucket.bucket_name))
self.logger.debug('fonts %s not matched for download in bucket %s' % (file_name, fonts_bucket.bucket_name))
def get_obj(self,
key,
dst,
use_resume=True,
part_size=(20 * 1024 * 1024),
num_threads=4):
"""
get files from oss,
:param key: oss key
:param dst: The path to save obj.
:return obj save path at last
"""
try:
if self.is_public_net:
raise Exception('Do not download from public')
if use_resume:
oss2.resumable_download(
self.bucket,
key,
dst,
store=oss2.ResumableDownloadStore(root='/tmp'),
multiget_threshold=20 * 1024 * 1024,
part_size=part_size,
num_threads=num_threads)
else:
self.bucket.get_object_to_file(key, dst)
return dst
except Exception as ex:
print ex.message
return None
def uploadFile(self, path, localPath):
oss2.resumable_upload(self._bucket,
path,
localPath,
store=oss2.ResumableDownloadStore(root='./tmp'),
multipart_threshold=20 * 1024 * 1024,
part_size=10 * 1024 * 1024,
num_threads=3)
def download(self, ossObject, loaclFile):
oss2.resumable_download(self.bucket, ossObject, loaclFile,
store=oss2.ResumableDownloadStore(root=os.path.dirname(loaclFile)),
multiget_threshold=1 * 1024,
part_size=10 * 1024 * 1024,
num_threads=3,
progress_callback= self.percentage
)
def download(self,url): if bucket.object_exists(url): local_f = "/data1/jenkins_dir/" + url oss2.resumable_download(bucket, url, local_f, store=oss2.ResumableDownloadStore(root='/tmp'), multiget_threshold=20*1024*1024, part_size=10*1024*1024, num_threads=2) return True else: return False
def OssDownload():
accessKeyId = sys.argv[1]
accessSecret = sys.argv[2]
EndPoint = sys.argv[3]
Bucket = sys.argv[4]
RemoteFile = sys.argv[5]
LocalFile = sys.argv[6]
auth = oss2.Auth(accessKeyId, accessSecret)
#endpoint = EndPoint
bucket = oss2.Bucket(auth, EndPoint, Bucket)
oss2.resumable_download(bucket,
RemoteFile,
LocalFile,
store=oss2.ResumableDownloadStore(root='/tmp'),
multiget_threshold=20 * 1024 * 1024,
part_size=10 * 1024 * 1024,
num_threads=3)
# -*- coding: utf-8 -*-
from __future__ import print_function
import os, sys
import oss2
# 百分比显示回调函数
def percentage(consumed_bytes, total_bytes):
if total_bytes:
rate = int(100 * (float(consumed_bytes)) / (float(total_bytes)))
print('\r{0}%'.format(rate), end=saveFile)
sys.stdout.flush()
auth = oss2.Auth('LTAI6KwW70DAfUyD', 'tXNFt4nfNkts2I91HJxe5WbFVWexDd')
bucket = oss2.Bucket(auth, 'oss-cn-hangzhou.aliyuncs.com', 'wordpress-liutingshu')
fileName = sys.argv[1]
saveFile = './' + fileName
oss2.resumable_download(bucket, fileName, saveFile,
store = oss2.ResumableDownloadStore(root='/tmp'),
multiget_threshold = 20 * 1024 * 1024,
part_size = 10 * 1024 * 1024,
num_threads = 5,
progress_callback = percentage)
print('\rDownload %s to %s Success!' % (fileName, saveFile))