def upload_to_obs():
# get the bucket name and OBS model path
obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
bucket_name = obs_path.split("/", 1)[0]
work_metric = obs_path.split("/", 1)[1] + '/'
work_model = obs_path.split("/", 1)[1] + '/model/'
# build bucket path
model_file = work_model + MODEL_NAME
config_file = work_model + CONFIG_NAME
metric_file = work_metric + METRIC_NAME
customize_service_file = work_model + CUSTOMIZE_SERVICE
obs_client.putContent(bucket_name, work_model, content=None)
# upload config file to OBS
print_title("upload config to obs !")
obs_client.putFile(bucket_name, config_file, file_path=LOCAL_CONFIG_PATH)
# upload custom service file to OBS
print_title("upload customize_service to obs !")
obs_client.putFile(bucket_name,
customize_service_file,
file_path=LOCAL_CUSTOMIZE_SERVICE_PATH)
# upload metric file to OBS
print_title("upload metric to obs !")
obs_client.putFile(bucket_name, metric_file, file_path=LOCAL_METRIC_PATH)
# upload model to OBS
print_title("upload model to obs !")
obs_client.putFile(bucket_name, model_file, file_path=LOCAL_MODEL_PATH)
return 0
def upload_to_obs():
obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
try:
bucket_name = obs_path.split("/", 1)[0]
work_metric = obs_path.split("/", 1)[1] + '/'
work_model = obs_path.split("/", 1)[1] + '/model/'
model_file = work_model + MODEL_NAME
config_file = work_model + CONFIG_NAME
metric_file = work_metric + METRIC_NAME
customize_service_file = work_model + CUSTOMIZE_SERVICE
obs_client.putContent(bucket_name, work_model, content=None)
# upload metric file to OBS
print_title("upload model to obs !")
obs_client.putFile(bucket_name, model_file, file_path=LOCAL_MODEL_PATH)
# upload config file to OBS
print_title("upload config to obs !")
obs_client.putFile(bucket_name,
config_file,
file_path=LOCAL_CONFIG_PATH)
# upload metric file to OBS
print_title("upload metric to obs !")
obs_client.putFile(bucket_name,
metric_file,
file_path=LOCAL_METRIC_PATH)
# upload custom service file to OBS
if os.path.exists(LOCAL_CUSTOMIZE_SERVICE_PATH):
print_title("upload customize_service to obs !")
obs_client.putFile(bucket_name,
customize_service_file,
file_path=LOCAL_CUSTOMIZE_SERVICE_PATH)
else:
print("user own customize_service.py not exists")
finally:
obs_client.close()
return 0
def upload_to_obs():
TestObs = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
bucketName = obs_path.split("/",1)[0]
workmetric = obs_path.split("/",1)[1] + '/'
workmodel = obs_path.split("/",1)[1] + '/model/'
#workconfig = obs_path.split("/",1)[1] + '/config/'
filemodel = workmodel + MODEL_NAME
fileconfig = workmodel + CONFIG_NAME
filemetric = workmetric + METRIC_NAME
#resultFileName = obs_path.split("/",1)[1] + '/model/xgboost.m'
#configName = obs_path.split("/",1)[1] + '/config/config.json'
TestObs.putContent(bucketName, workmodel, content=None)
#TestObs.putContent(bucketName, workconfig, content=None)
print_title("upload model to obs !")
TestObs.putFile(bucketName, filemodel, file_path=LOCAL_MODEL_DIR)
print_title("upload config to obs !")
TestObs.putFile(bucketName, fileconfig, file_path=LOCAL_CONFIG_DIR)
print_title("upload metric to obs !")
TestObs.putFile(bucketName, filemetric, file_path=LOCAL_METRIC_DIR)
return 0
def create_model_dir():
obsClient = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
bucketName = obs_path.split("/", 1)[0]
workDir = obs_path.split("/", 1)[1] + '/model/'
resp_dir = obsClient.putContent(bucketName, workDir, content=None)
if resp_dir.status < 300:
print('Create workDir model success:', resp_dir.requestId)
return workDir
print('Fail to create workDir model and errorCode:', resp_dir.errorCode)
print('errorMessage:', resp_dir.errorMessage)
return ""
class ObsFeedStorage(BlockingFeedStorage):
"""
Feed exporter storage backend for huawei cloud OBS.
"""
def __init__(self, uri, access_key=None, secret_key=None, endpoint=None):
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or access_key
self.secret_key = u.password or secret_key
self.endpoint = endpoint
self.keyname = u.path[1:] # remove first "/"
if access_key is None or secret_key is None or endpoint is None:
raise NotConfigured(
'missing access_key or secret_key or endpoint for huawei cloud'
)
try:
from obs import ObsClient
except ImportError:
raise NotConfigured('missing esdk-obs-python library')
self.obs_client = ObsClient(access_key_id=self.access_key,
secret_access_key=self.secret_key,
server=self.endpoint)
@classmethod
def from_crawler(cls, crawler, uri):
return cls(
uri=uri,
access_key=crawler.settings['HUAWEI_ACCESS_KEY_ID'],
secret_key=crawler.settings['HUAWEI_SECRET_ACCESS_KEY'],
endpoint=crawler.settings['HUAWEI_OBS_ENDPOINT'],
)
@classmethod
def from_settings(cls, settings, uri):
return cls(
uri=uri,
access_key=settings['HUAWEI_ACCESS_KEY_ID'],
secret_key=settings['HUAWEI_SECRET_ACCESS_KEY'],
endpoint=settings['HUAWEI_OBS_ENDPOINT'],
)
def _store_in_thread(self, file):
file.seek(0)
resp = self.obs_client.putContent(self.bucketname, self.keyname, file)
if resp.status >= 300:
raise IOError('{}: {}'.format(resp.errorCode, resp.errorMessage))
class HWObsClient(object):
def __init__(self):
self.key = ""
self.secret = ""
self.obsclient = ObsClient(
access_key_id=self.key,
secret_access_key=self.secret,
server='',
)
def streamupload(self, objectname, filestream, bucketname="img-tmp"):
resp = self.obsclient.putContent(bucketname, objectname, content=filestream) # filestream为上传的文件数据流
if resp.status < 300:
print('requestId:', resp.requestId)
return True
else:
print('errorCode:', resp.errorCode)
print('errorMessage:', resp.errorMessage)
return False
# Constructs a obs client instance with your account for accessing OBS
obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server)
# Create bucket
print('Create a new bucket for demo\n')
obsClient.createBucket(bucketName)
# Setting object mime type
headers = PutObjectHeader(contentType='text/plain')
# Setting self-defined metadata
metadata = {'meta1': 'value1', 'meta2': 'value2'}
resp = obsClient.putContent(bucketName,
objectKey,
'Hello OBS',
metadata=metadata,
headers=headers)
if resp.status < 300:
print('Create object ' + objectKey + ' successfully!\n')
else:
print('common msg:status:', resp.status, ',errorCode:', resp.errorCode,
',errorMessage:', resp.errorMessage)
# Get object metadata
resp = obsClient.getObjectMetadata(bucketName, objectKey)
header = dict(resp.header)
print('\tContentType:' + header.get('content-type'))
print('\tmeta1:' + header.get('meta1'))
print('\tmeta2:' + header.get('meta2'))
token = get_token(domain_name, project_id, user=user, password=password)
# get ak/sk credentials
credentials = create_temporary_ak_sk(token)
AK = credentials["access"]
SK = credentials["secret"]
# Constructs a obs client instance with your account for accessing OBS
obs_client = ObsClient(access_key_id=AK,
secret_access_key=SK,
server=server)
bucket_client = obs_client.bucketClient(bucket_name)
# TODO create OBS folders (not working)
base_folder = f"{uid}/"
obs_client.putContent(bucket_name, base_folder)
resp = obs_client.getObjectMetadata(bucket_name, base_folder)
print('Size of the empty folder ' + base_folder + ' is ' +
str(dict(resp.header).get('content-length')))
code_folder = f"{base_folder}code/"
obs_client.putContent(bucket_name, code_folder)
data_folder = f"{base_folder}data/"
obs_client.putContent(bucket_name, data_folder)
model_folder = f"{base_folder}model/"
obs_client.putContent(bucket_name, model_folder)
# upload script and data to OBS
train_file = code_folder + "train.py"
obs_client.putFile(bucket_name, train_file,
# Create bucket
print('Create a new bucket for demo\n')
obsClient.createBucket(bucketName)
content = 'Hello OBS'
keyPrefix = 'MyObjectKey'
folderPrefix = 'src'
subFolderPrefix = 'test'
keys = []
# First prepare folders and sub folders
for i in range(5):
key = folderPrefix + str(i) + '/'
obsClient.putContent(bucketName, key)
keys.append(Object(key))
for j in range(3):
subKey = key + subFolderPrefix + str(j) + '/'
obsClient.putContent(bucketName, subKey)
keys.append(Object(subKey))
# Insert 2 objects in each folder
resp = obsClient.listObjects(bucketName)
for content in resp.body.contents:
for i in range(2):
objectKey = content.key + keyPrefix + str(i)
obsClient.putContent(bucketName, objectKey, content)
keys.append(Object(objectKey))
# Constructs a obs client instance with your account for accessing OBS
obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server)
# Create bucket
print('Create a new bucket for demo\n')
obsClient.createBucket(bucketName)
content = 'Hello OBS'
keyPrefix = 'MyObjectKey'
keys = []
# First insert 100 objects for demo
for i in range(100):
key = keyPrefix + str(i)
obsClient.putContent(bucketName, key, content)
keys.append(Object(key))
print('Put ' + str(len(keys)) + ' objects completed.' + '\n')
# List objects using default parameters, will return up to 1000 objects
print('List objects using default parameters:\n')
resp = obsClient.listObjects(bucketName)
for content in resp.body.contents:
print('\t' + content.key + ' etag[' + content.etag + ']')
print('\n')
# List the first 10 objects
print('List the first 10 objects :\n')
resp = obsClient.listObjects(bucketName, max_keys=10)
from __future__ import print_function
from obs import ObsClient
AK = '*** Provide your Access Key ***'
SK = '*** Provide your Secret Key ***'
server = 'https://your-endpoint'
bucketName = 'my-obs-bucket-demo'
# Constructs a obs client instance with your account for accessing OBS
obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server)
# Create bucket
print('Create a new bucket for demo\n')
obsClient.createBucket(bucketName)
keySuffixWithSlash1 = 'MyObjectKey1/'
keySuffixWithSlash2 = 'MyObjectKey2/'
# Create two empty folder without request body, note that the key must be suffixed with a slash
obsClient.putContent(bucketName, keySuffixWithSlash1, '')
print('Creating an empty folder ' + keySuffixWithSlash1)
obsClient.putContent(bucketName, keySuffixWithSlash2)
print('Creating an empty folder ' + keySuffixWithSlash2)
# Verify whether the size of the empty folder is zero
resp = obsClient.getObjectMetadata(bucketName, keySuffixWithSlash1)
print('Size of the empty folder ' + keySuffixWithSlash1 + ' is ' + str(dict(resp.header).get('content-length')))
resp = obsClient.getObjectMetadata(bucketName, keySuffixWithSlash2)
print('Size of the empty folder ' + keySuffixWithSlash2 + ' is ' + str(dict(resp.header).get('content-length')))
def obs_uplodfile(ak, sk, server, bucketname, localdir, obsdir_parent):
'''
:param ak: obs的Access Key
:param sk: obs的Secret Key
:param server: obs的endpoint地址
:param bucketname: obs的名字
:param localdir: 需要上传到obs本地目录
:param obsdir_parent: 希望上传到obs后的目录名字 比如这里设置成mongodb_backup最终的备份目录为： mongodb_backup/2019-07/2019-07-08/
:return:
'''
from obs import ObsClient
import os
import time
# 创建ObsClient实例
obsClient = ObsClient(access_key_id=ak,
secret_access_key=sk,
server=server)
DATE = time.strftime("%Y%m%d")
MONTH = time.strftime("%Y%m")
# 本地需要上传的目录 确保这个目录下都是文件 子目录不支持
localdir = localdir
# bucketname
bucketname = bucketname
Logger().log('开始上传到 %s OBS ...' % bucketname)
# 设置分段上传时的最大并发数
taskNum = 5
# 设置分段大小为10MB
partSize = 10 * 1024 * 1024
# 开启断点续传模式
enableCheckpoint = True
# obs上存放数据的文件夹
obsdir_parent = obsdir_parent
obsdir = obsdir_parent + '/' + MONTH + '/' + DATE # mongodb_backup/2019-07/2019-07-08/
# 规范整理文件名和文件绝对路径
filename_list = []
abspath_list = []
for dir, _, file in os.walk(localdir): # 变量解压
filename_list = file
for f in file:
abspath = dir + "/" + f
abspath_list.append(abspath)
filename_abspath_list = zip(filename_list, abspath_list)
for t in filename_abspath_list:
# 设置待上传的本地文件，需要指定到具体的文件名
uploadFile = t[1]
objectName = t[0]
# 进行断点续传上传
try:
# 创建obs存放数据的文件夹
cdir = obsClient.putContent(bucketname, obsdir, content=None)
obs_filepath = os.path.join(obsdir, objectName)
resp = obsClient.uploadFile(bucketname, obs_filepath, uploadFile,
partSize, taskNum, enableCheckpoint)
Logger().log('%s开始上传' % objectName)
if resp.status < 300:
Logger().log('%s上传成功\n' % objectName)
# content='status:', resp['status'], 'filename:', resp['body']['key']
else:
Logger().log('%s上传失败' % objectName, mode=False)
#content= 'errorCode:', resp.errorCode, 'errorMessage:', resp.errorMessage
# 操作失败时可再次调用断点续传上传接口进行重新上传
except Exception as e:
Logger().log('%s上传失败,发生错误%s' % (objectName, e), mode=False)
SK = '*** Provide your Secret Key ***'
server = 'https://your-endpoint'
bucketName = 'my-obs-cold-bucket-demo'
objectKey = 'my-obs-cold-object-key-demo'
# Constructs a obs client instance with your account for accessing OBS
obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server)
# Create a cold bucket
print('Create a new cold bucket for demo\n')
obsClient.createBucket(bucketName,
CreateBucketHeader(storageClass=StorageClass.COLD))
# Create a cold object
print('Create a new cold object for demo\n')
obsClient.putContent(bucketName, objectKey, 'Hello OBS')
# Restore the cold object
print('Restore the cold object')
obsClient.restoreObject(bucketName, objectKey, 1, tier=RestoreTier.EXPEDITED)
# Wait 6 minute to get the object
time.sleep(6 * 60)
# Get the cold object status
print('Get the cold object status')
resp = obsClient.restoreObject(bucketName,
objectKey,
1,
tier=RestoreTier.EXPEDITED)
print('\tstatus code ' + str(resp.status))
