def resumableUpload(self, path):
"""断点续传上传
Args:
path (TYPE): file abspath
Returns:
TYPE: Description
"""
part_size = os.path.getsize(path) if os.path.getsize(
path) < 1024 * 1024 else os.path.getsize(path) // 10
success = False
retry = 10
while not success and retry > 0:
retry -= 1
try:
oss2.resumable_upload(
self.bucket,
path.rsplit(os.sep, 1)[1],
path,
progress_callback=self.percentage,
# store=oss2.ResumableStore(root='/tmp'),
store=oss2.ResumableStore(root='/tmp' if checkOS() ==
'linux' else config.BASE_DIR),
multipart_threshold=1024 * 1024,
part_size=part_size,
num_threads=4)
success = True
return True
except oss2.exceptions.RequestError as e:
log.warn('上传失败，即将进行重试')
time.sleep(2)
continue
return False
def _do(self, job):
config = Config()
if job.action == _Job.PUSH:
encode_md5 = base64.b64encode(bytearray.fromhex(job.md5)).decode()
headers = {
"Content-MD5": encode_md5,
snapshot.AliOssSnapshot.meta_md5: job.md5
}
try:
oss2.resumable_upload(
self.target_snapshot.bucket,
job.target,
job.src,
headers=headers,
store=oss2.ResumableStore(root=config.cache_dir),
multipart_threshold=config.multipart_threshold,
part_size=config.multipart_threshold,
num_threads=config.num_threads)
except oss2.exceptions.InvalidDigest:
job.info = "md5 mismatch"
raise JobError
elif job.action == _Job.REMOVE:
self.target_snapshot.bucket.delete_object(job.target)
def put_obj(self,
key,
src,
use_resume=True,
part_size=(20 * 1024 * 1024),
num_threads=4):
"""
put file to oss
:param key:
:param src:
"""
# use resume
try:
if use_resume:
oss2.resumable_upload(self.bucket,
key,
src,
store=oss2.ResumableStore(root='/tmp'),
multipart_threshold=100 * 1024,
part_size=part_size,
num_threads=num_threads)
else:
self.bucket.put_object_from_file(key, src)
except Exception as ex:
print ex.message
def _push(self, fi):
try:
headers = ALI.META
headers.update({"Content-Length": str(fi.size)}) # for cdn check
headers.update({"Content-MD5": fi.md5}) # for cdn check
headers.update({"x-oss-meta-etag": fi.etag})
ret = oss2.resumable_upload(self.bucket_mgr,
fi.rname,
fi.rpath,
store=oss2.ResumableStore(root='/tmp'),
multipart_threshold=10 * 1024 * 1024,
part_size=10 * 1024 * 1024,
num_threads=4,
headers=headers)
if ret.status != 200:
log.error("ALI:_push: {file} error".format(file=fi.rpath))
raise Exception(
"ALI:_push: {file} error".format(file=fi.rpath))
if ret.crc != fi.crc: # local check
log.error(
"ALI:_push check: {file} error".format(file=fi.rpath))
raise Exception(
"ALI:_push check: {file} error".format(file=fi.rpath))
log.info("{fi} distributed".format(fi=fi))
self.upload_files.add(fi.rname)
except Exception as e:
log.error("ALI:_push error: {error}".format(error=str(e)))
raise Exception(
"Distribute_ALI_push_error :{error}".format(error=str(e)))
def upload_files(self, objname, localfile):
oss2.resumable_upload(
self.Bucket,
objname,
localfile,
store=oss2.ResumableStore(
root=current_app.config['UPLOADED_FILE_DEST']))
def upload_fonts(self, file_path):
file_name = file_path.split('/')[-1]
fonts_buckets = self.get_font_bucket()
for fonts_bucket in fonts_buckets:
# get bucket region
fonts_bucket_info = fonts_bucket.get_bucket_info()
region = fonts_bucket_info.location.split('-')[-1]
# get font bucket sotorage dir
fonts_dir = config.ali_fonts_bucket[region]['font_dir']
upload_res = oss2.resumable_upload(fonts_bucket, os.path.join(fonts_dir, file_name), file_path,
store=oss2.ResumableStore(root='./tmp_files/uploads'),
multipart_threshold=100 * 1024,
part_size=100 * 1024,
progress_callback=self.percentage,
num_threads=4)
print('', end='\n')
# print('Upload response: %s' % upload_res)
if upload_res.status == 200 or upload_res.resp.status == "OK":
print('Font %s upload to bucket %s successed' % (file_name, fonts_bucket.bucket_name))
self.logger.info('Font %s upload to bucket %s successed' % (file_name, fonts_bucket.bucket_name))
else:
print('Font %s upload to bucket %s failed' % (file_name, fonts_bucket.bucket_name))
self.logger.error('Font %s upload to bucket %s failed' % (file_name, fonts_bucket.bucket_name))
def multi_upload_obj(self, remote_file, local_file):
oss2.resumable_upload(
self._bucket,
remote_file,
local_file,
store=oss2.ResumableStore(root='/tmp'), #指定保存断点信息的目录
multipart_threshold=100 * 1024, #文件长度大于该值时，则用分片上传
part_size=100 * 1024, #分片大小
num_threads=4) #并发上传的线程数
def upload(self,file):
with open(file,'r+') as f:
lines = f.readlines()
for line in lines:
line1 = line.strip('\n')
remote_file = line1.split('/data1/jenkins_dir/')
oss2.resumable_upload(bucket, remote_file[1], line1,
store=oss2.ResumableStore(root='/tmp'),
multipart_threshold=100*1024,
part_size=100*1024,
num_threads=4)
def test_resumable_store_dir(self):
root = "./"
store_dir = "test-resumable-store-dir"
path = root + store_dir
self.assertFalse(os.path.exists(path))
store = oss2.ResumableStore(root, store_dir)
self.assertTrue(os.path.exists(path))
self.assertTrue(os.path.isdir(path))
os.rmdir(path)
def resumable_upload_from_local(self,localfile=None,ossfile=None):
'''
'''
ossfile = os.path.join(self.root_name,ossfile)
result = oss2.resumable_upload(self.bucket,ossfile,localfile,
store=oss2.ResumableStore(root='/tmp'),
multipart_threshold=100*1024,
part_size=100*1024,
num_threads=4,
progress_callback=self.progress_callback
)
if result.status == 200:
url = os.path.join(self.domain,ossfile)
else:
url = ''
return url
def resume_upload(self,
remote_file_path,
local_file_path,
multipart_threshold=100 * 1024,
part_size=100 * 1024,
num_threads=5,
progress_callback=None):
name = local_file_path.split('/')[-1]
content_type = mimetypes.guess_type(
name)[0] or 'application/x-octet-stream'
with open(local_file_path, 'rb') as f:
content_file = File(f)
if hasattr(content_file, 'chunks'):
content = b''.join(content_file.chunks())
else:
content = content_file.read()
if not content:
raise AliyunOSSException(
'`local_file_path` not exists error, local_file_path: {}'.
format(local_file_path))
content_len = str(len(content))
headers = {
'x-oss-acl': self.acl,
'Content-Type': content_type,
'Content-Length': content_len,
}
return oss2.resumable_upload(self.bucket,
remote_file_path,
local_file_path,
store=oss2.ResumableStore(root='/tmp'),
headers=headers,
multipart_threshold=multipart_threshold,
part_size=part_size,
num_threads=num_threads,
progress_callback=progress_callback)
print('Backup App success!')
bucket = oss2.Bucket(auth, endpoint, bucketName) #?~N??~O~V桶
def percentage(consumed_bytes, total_bytes):
if total_bytes:
rate = int(100 * (float(consumed_bytes) / float(total_bytes)))
print('\rAre uploading:{0}%'.format(rate), end='')
sys.stdout.flush()
#currentTime = time.strftime('%Y-%m-%d_%H:%M_timestamp:%s',time.localtime(time.time()))
#fileName = currentTime+'.sql' #?~K??~N??~P~N?~@
#fileName = 'redmine_bk'+fileName #?~K??~N??~I~M?~@
#fileName = 'sql/'+fileName
ossAppFilePath = 'application/' + appFileName
print(ossAppFilePath)
oss2.resumable_upload(
bucket,
ossAppFilePath,
appFilePath,
store=oss2.ResumableStore(root='/tmp'),
multipart_threshold=100 * 1024,
part_size=100 * 1024,
num_threads=4,
headers={"Content-Type": "application/octet-stream; charset=utf-8"},
progress_callback=percentage) #?~V??~B?�| ?~J?| ?~V??~H?~L?~X?示?~[度
def upload(file, name, range, description, proxy):
if not proxy:
os.environ['HTTP_PROXY'] = ''
os.environ['HTTPS_PROXY'] = ''
os.environ['http_proxy'] = ''
os.environ['https_proxy'] = ''
if not (file.endswith(".zip") or file.endswith(".tar.gz")):
sys.exit('Error: uploading file should be one of .zip or .tar.gz type')
filepath = Path(file)
total_size = filepath.stat().st_size
if not filepath.exists():
sys.exit('Error: file not exists')
name = name or filepath.name
sha1 = hashlib.sha1()
sha1.update((f"{filepath.name}-{total_size}").encode())
digest = sha1.hexdigest()
dataset_file = Path.home() / '.featurize' / f"file_upload_checkpoint_{digest}" / "dataset.json"
dataset_file.parent.mkdir(parents=True, exist_ok=True)
try:
dataset = json.loads(dataset_file.read_text())
except (FileNotFoundError, json.JSONDecodeError):
# init dataset
res = client.dataset.create(name, range, description)
dataset = {
'id': res['id'],
'dataset_center': res['dataset_center'],
'uploader_id': res['uploader_id'],
'consumed_bytes': 0
}
dataset_file.write_text(json.dumps(dataset))
credential = client.oss_credential.get()
auth = oss2.StsAuth(
credential['AccessKeyId'],
credential['AccessKeySecret'],
credential['SecurityToken']
)
bucket = oss2.Bucket(auth, 'http://oss-cn-beijing.aliyuncs.com', dataset['dataset_center']['bucket'])
def progress_callback(consumed_bytes, total_bytes):
pbar.update(consumed_bytes - pbar.n)
dataset['consumed_bytes'] = consumed_bytes
dataset_file.write_text(json.dumps(dataset))
pbar = tqdm(total=total_size, unit='B', unit_scale=True)
path = f"{dataset['uploader_id']}_{dataset['id']}/{filepath.name}"
result = oss2.resumable_upload(
bucket,
path,
filepath.resolve().as_posix(),
store=oss2.ResumableStore(root='/tmp'),
multipart_threshold=8 * 1024 * 1024,
part_size=1024 * 1024 * 1,
num_threads=1,
progress_callback=progress_callback
)
if result.status != 200:
sys.exit(f"Error: upload respond with code {result.status}")
client.dataset.update(
dataset_id=dataset['id'],
uploaded=True,
domain=f"{dataset['dataset_center']['bucket']}.oss-cn-beijing.aliyuncs.com",
path=path,
size=total_size,
filename=filepath.name
)
shutil.rmtree(dataset_file.parent)
def encrypt_and_upload_files(self,
local_file_name,
remote_object_name,
storage_class='Standard',
file_sha256=None,
cache_control='no-store',
compare_sha256_before_uploading=False):
"""使用KMS加密并上传文件
Args:
local_file_name (str): 本地文件路径
remote_object_name (str): 远程文件路径
storage_class (str, 可选): Object的存储类型，取值：Standard、IA、Archive和ColdArchive。默认值为Standard
file_sha256 (str, 可选): 如不提供将会自动计算本地文件sha256
cache_control (str, 可选)
compare_sha256_before_uploading (bool, 可选): 是否在上传之前对比远端文件的sha256，如相同则跳过上传
"""
if not file_sha256:
file_sha256 = calculate_local_file_sha256(local_file_name)
retry_count = 0
if compare_sha256_before_uploading:
try:
remote_object_sha256 = self.get_remote_file_headers(
remote_object_name)['x-oss-meta-sha256']
except:
remote_object_sha256 = file_sha256
if remote_object_sha256 == file_sha256:
logger.info("[encrypt_and_upload_files]sha256相同，跳过%s文件的上传" %
local_file_name)
return 200
while True:
try:
retry_count += 1
oss2.resumable_upload(
self.__bucket,
remote_object_name,
local_file_name,
store=oss2.ResumableStore(root=config.temp_dir),
multipart_threshold=1024 * 1024 * 50,
part_size=1024 * 1024 * 50,
num_threads=4,
headers={
"content-length":
str(os.path.getsize(local_file_name)),
"Cache-Control": cache_control,
"x-oss-server-side-encryption": "KMS",
"x-oss-storage-class": storage_class,
"x-oss-meta-sha256": file_sha256
})
break
except (oss2.exceptions.ClientError, oss2.exceptions.RequestError,
ConnectionResetError) as err:
if retry_count < config.Max_Retries:
logger.error(
"[encrypt_and_upload_files] error, retrying time %d" %
retry_count)
logger.error(err)
else:
logger.exception("[encrypt_and_upload_files] Error")
raise oss2.exceptions.RequestError
sleep(square(retry_count) * 10)
while subprocess.run(self.__ping_cmd,
capture_output=True).returncode != 0:
logger.error("无法连接网络，10秒后重试")
sleep(10)
return 200
def upload_files(self, objname, localfile):
oss2.resumable_upload(
self.Bucket,
objname,
localfile,
store=oss2.ResumableStore(root='/Users/root1/webdev/fileload'))
def push_to_oss(self, package_name=None):
pkg_list = self.fetch_package_list()
pkg_dir = os.path.join(self.base_path, 'packages')
if not os.path.exists(pkg_dir):
os.mkdir(pkg_dir)
print('\n\nSyncing packages to OSS')
oss_auth = oss2.Auth(self.access_key_id, self.access_key_secret)
bucket = oss2.Bucket(oss_auth, self.endpoint, self.bucket)
for pkg in pkg_list:
if package_name:
if package_name == pkg.file_name:
print('downloading {0}'.format(pkg.file_name))
r = requests.get(pkg.url, stream=True)
f = open(os.path.join(pkg_dir, pkg.file_name), 'wb')
for chunk in r.iter_content(chunk_size=512):
if chunk:
f.write(chunk)
f.close()
print('downloaded {0}'.format(pkg.file_name))
# upload to oss
sleep(10)
print('pushing {0}'.format(pkg.file_name))
with open(os.path.join(self.base_path, 'packages', pkg.file_name), 'rb') as fileobj:
bucket.put_object(pkg.file_name, fileobj)
bucket.put_object_acl(pkg.file_name, oss2.OBJECT_ACL_PUBLIC_READ)
sys.exit(0)
continue
else:
if not bucket.object_exists(pkg.file_name):
print('remote: {0} is not exist'.format(pkg))
# check local package
if os.path.exists(os.path.join(pkg_dir, pkg.file_name)):
print('local: {0} is exist'.format(pkg))
print('pushing {0}'.format(pkg.file_name))
oss2.resumable_upload(bucket, pkg.file_name,
os.path.join(self.base_path, 'packages', pkg.file_name),
store=oss2.ResumableStore(root='/tmp'),
multipart_threshold=100 * 1024,
part_size=100 * 1024,
num_threads=4,
progress_callback=percentage)
bucket.put_object_acl(pkg.file_name, oss2.OBJECT_ACL_PUBLIC_READ)
os.remove(os.path.join(self.base_path, 'packages', pkg.file_name))
elif pkg.platform == 'linux_x86-x86':
pass
else:
print('local: {0} is not exist'.format(pkg))
print('downloading {0}'.format(pkg.file_name))
r = requests.get(pkg.url, stream=True)
f = open(os.path.join(pkg_dir, pkg.file_name), 'wb')
for chunk in r.iter_content(chunk_size=512):
if chunk:
f.write(chunk)
f.close()
# upload to oss
sleep(2)
print('pushing {0}'.format(pkg.file_name))
package_dst = os.path.join(self.base_path, 'packages', pkg.file_name)
oss2.resumable_upload(bucket, pkg.file_name, package_dst,
store=oss2.ResumableStore(root='/tmp'),
multipart_threshold=100 * 1024,
part_size=100 * 1024,
num_threads=4,
progress_callback=percentage)
bucket.put_object_acl(pkg.file_name, oss2.OBJECT_ACL_PUBLIC_READ)
os.remove(os.path.join(self.base_path, 'packages', pkg.file_name))
else:
print('remote: {0} is exist'.format(pkg))
