def upload_to_obs(): # get the bucket name and OBS model path obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) bucket_name = obs_path.split("/", 1)[0] work_metric = obs_path.split("/", 1)[1] + '/' work_model = obs_path.split("/", 1)[1] + '/model/' # build bucket path model_file = work_model + MODEL_NAME config_file = work_model + CONFIG_NAME metric_file = work_metric + METRIC_NAME customize_service_file = work_model + CUSTOMIZE_SERVICE obs_client.putContent(bucket_name, work_model, content=None) # upload config file to OBS print_title("upload config to obs !") obs_client.putFile(bucket_name, config_file, file_path=LOCAL_CONFIG_PATH) # upload custom service file to OBS print_title("upload customize_service to obs !") obs_client.putFile(bucket_name, customize_service_file, file_path=LOCAL_CUSTOMIZE_SERVICE_PATH) # upload metric file to OBS print_title("upload metric to obs !") obs_client.putFile(bucket_name, metric_file, file_path=LOCAL_METRIC_PATH) # upload model to OBS print_title("upload model to obs !") obs_client.putFile(bucket_name, model_file, file_path=LOCAL_MODEL_PATH) return 0
def upload_to_obs(): obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) try: bucket_name = obs_path.split("/", 1)[0] work_metric = obs_path.split("/", 1)[1] + '/' work_model = obs_path.split("/", 1)[1] + '/model/' model_file = work_model + MODEL_NAME config_file = work_model + CONFIG_NAME metric_file = work_metric + METRIC_NAME customize_service_file = work_model + CUSTOMIZE_SERVICE obs_client.putContent(bucket_name, work_model, content=None) # upload metric file to OBS print_title("upload model to obs !") obs_client.putFile(bucket_name, model_file, file_path=LOCAL_MODEL_PATH) # upload config file to OBS print_title("upload config to obs !") obs_client.putFile(bucket_name, config_file, file_path=LOCAL_CONFIG_PATH) # upload metric file to OBS print_title("upload metric to obs !") obs_client.putFile(bucket_name, metric_file, file_path=LOCAL_METRIC_PATH) # upload custom service file to OBS if os.path.exists(LOCAL_CUSTOMIZE_SERVICE_PATH): print_title("upload customize_service to obs !") obs_client.putFile(bucket_name, customize_service_file, file_path=LOCAL_CUSTOMIZE_SERVICE_PATH) else: print("user own customize_service.py not exists") finally: obs_client.close() return 0
def upload_to_obs(): TestObs = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) bucketName = obs_path.split("/",1)[0] workmetric = obs_path.split("/",1)[1] + '/' workmodel = obs_path.split("/",1)[1] + '/model/' #workconfig = obs_path.split("/",1)[1] + '/config/' filemodel = workmodel + MODEL_NAME fileconfig = workmodel + CONFIG_NAME filemetric = workmetric + METRIC_NAME #resultFileName = obs_path.split("/",1)[1] + '/model/xgboost.m' #configName = obs_path.split("/",1)[1] + '/config/config.json' TestObs.putContent(bucketName, workmodel, content=None) #TestObs.putContent(bucketName, workconfig, content=None) print_title("upload model to obs !") TestObs.putFile(bucketName, filemodel, file_path=LOCAL_MODEL_DIR) print_title("upload config to obs !") TestObs.putFile(bucketName, fileconfig, file_path=LOCAL_CONFIG_DIR) print_title("upload metric to obs !") TestObs.putFile(bucketName, filemetric, file_path=LOCAL_METRIC_DIR) return 0
def create_model_dir(): obsClient = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) bucketName = obs_path.split("/", 1)[0] workDir = obs_path.split("/", 1)[1] + '/model/' resp_dir = obsClient.putContent(bucketName, workDir, content=None) if resp_dir.status < 300: print('Create workDir model success:', resp_dir.requestId) return workDir print('Fail to create workDir model and errorCode:', resp_dir.errorCode) print('errorMessage:', resp_dir.errorMessage) return ""
class ObsFeedStorage(BlockingFeedStorage): """ Feed exporter storage backend for huawei cloud OBS. """ def __init__(self, uri, access_key=None, secret_key=None, endpoint=None): u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key self.secret_key = u.password or secret_key self.endpoint = endpoint self.keyname = u.path[1:] # remove first "/" if access_key is None or secret_key is None or endpoint is None: raise NotConfigured( 'missing access_key or secret_key or endpoint for huawei cloud' ) try: from obs import ObsClient except ImportError: raise NotConfigured('missing esdk-obs-python library') self.obs_client = ObsClient(access_key_id=self.access_key, secret_access_key=self.secret_key, server=self.endpoint) @classmethod def from_crawler(cls, crawler, uri): return cls( uri=uri, access_key=crawler.settings['HUAWEI_ACCESS_KEY_ID'], secret_key=crawler.settings['HUAWEI_SECRET_ACCESS_KEY'], endpoint=crawler.settings['HUAWEI_OBS_ENDPOINT'], ) @classmethod def from_settings(cls, settings, uri): return cls( uri=uri, access_key=settings['HUAWEI_ACCESS_KEY_ID'], secret_key=settings['HUAWEI_SECRET_ACCESS_KEY'], endpoint=settings['HUAWEI_OBS_ENDPOINT'], ) def _store_in_thread(self, file): file.seek(0) resp = self.obs_client.putContent(self.bucketname, self.keyname, file) if resp.status >= 300: raise IOError('{}: {}'.format(resp.errorCode, resp.errorMessage))
class HWObsClient(object): def __init__(self): self.key = "" self.secret = "" self.obsclient = ObsClient( access_key_id=self.key, secret_access_key=self.secret, server='', ) def streamupload(self, objectname, filestream, bucketname="img-tmp"): resp = self.obsclient.putContent(bucketname, objectname, content=filestream) # filestream为上传的文件数据流 if resp.status < 300: print('requestId:', resp.requestId) return True else: print('errorCode:', resp.errorCode) print('errorMessage:', resp.errorMessage) return False
# Constructs a obs client instance with your account for accessing OBS obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) # Create bucket print('Create a new bucket for demo\n') obsClient.createBucket(bucketName) # Setting object mime type headers = PutObjectHeader(contentType='text/plain') # Setting self-defined metadata metadata = {'meta1': 'value1', 'meta2': 'value2'} resp = obsClient.putContent(bucketName, objectKey, 'Hello OBS', metadata=metadata, headers=headers) if resp.status < 300: print('Create object ' + objectKey + ' successfully!\n') else: print('common msg:status:', resp.status, ',errorCode:', resp.errorCode, ',errorMessage:', resp.errorMessage) # Get object metadata resp = obsClient.getObjectMetadata(bucketName, objectKey) header = dict(resp.header) print('\tContentType:' + header.get('content-type')) print('\tmeta1:' + header.get('meta1')) print('\tmeta2:' + header.get('meta2'))
token = get_token(domain_name, project_id, user=user, password=password) # get ak/sk credentials credentials = create_temporary_ak_sk(token) AK = credentials["access"] SK = credentials["secret"] # Constructs a obs client instance with your account for accessing OBS obs_client = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) bucket_client = obs_client.bucketClient(bucket_name) # TODO create OBS folders (not working) base_folder = f"{uid}/" obs_client.putContent(bucket_name, base_folder) resp = obs_client.getObjectMetadata(bucket_name, base_folder) print('Size of the empty folder ' + base_folder + ' is ' + str(dict(resp.header).get('content-length'))) code_folder = f"{base_folder}code/" obs_client.putContent(bucket_name, code_folder) data_folder = f"{base_folder}data/" obs_client.putContent(bucket_name, data_folder) model_folder = f"{base_folder}model/" obs_client.putContent(bucket_name, model_folder) # upload script and data to OBS train_file = code_folder + "train.py" obs_client.putFile(bucket_name, train_file,
# Create bucket print('Create a new bucket for demo\n') obsClient.createBucket(bucketName) content = 'Hello OBS' keyPrefix = 'MyObjectKey' folderPrefix = 'src' subFolderPrefix = 'test' keys = [] # First prepare folders and sub folders for i in range(5): key = folderPrefix + str(i) + '/' obsClient.putContent(bucketName, key) keys.append(Object(key)) for j in range(3): subKey = key + subFolderPrefix + str(j) + '/' obsClient.putContent(bucketName, subKey) keys.append(Object(subKey)) # Insert 2 objects in each folder resp = obsClient.listObjects(bucketName) for content in resp.body.contents: for i in range(2): objectKey = content.key + keyPrefix + str(i) obsClient.putContent(bucketName, objectKey, content) keys.append(Object(objectKey))
# Constructs a obs client instance with your account for accessing OBS obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) # Create bucket print('Create a new bucket for demo\n') obsClient.createBucket(bucketName) content = 'Hello OBS' keyPrefix = 'MyObjectKey' keys = [] # First insert 100 objects for demo for i in range(100): key = keyPrefix + str(i) obsClient.putContent(bucketName, key, content) keys.append(Object(key)) print('Put ' + str(len(keys)) + ' objects completed.' + '\n') # List objects using default parameters, will return up to 1000 objects print('List objects using default parameters:\n') resp = obsClient.listObjects(bucketName) for content in resp.body.contents: print('\t' + content.key + ' etag[' + content.etag + ']') print('\n') # List the first 10 objects print('List the first 10 objects :\n') resp = obsClient.listObjects(bucketName, max_keys=10)
from __future__ import print_function from obs import ObsClient AK = '*** Provide your Access Key ***' SK = '*** Provide your Secret Key ***' server = 'https://your-endpoint' bucketName = 'my-obs-bucket-demo' # Constructs a obs client instance with your account for accessing OBS obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) # Create bucket print('Create a new bucket for demo\n') obsClient.createBucket(bucketName) keySuffixWithSlash1 = 'MyObjectKey1/' keySuffixWithSlash2 = 'MyObjectKey2/' # Create two empty folder without request body, note that the key must be suffixed with a slash obsClient.putContent(bucketName, keySuffixWithSlash1, '') print('Creating an empty folder ' + keySuffixWithSlash1) obsClient.putContent(bucketName, keySuffixWithSlash2) print('Creating an empty folder ' + keySuffixWithSlash2) # Verify whether the size of the empty folder is zero resp = obsClient.getObjectMetadata(bucketName, keySuffixWithSlash1) print('Size of the empty folder ' + keySuffixWithSlash1 + ' is ' + str(dict(resp.header).get('content-length'))) resp = obsClient.getObjectMetadata(bucketName, keySuffixWithSlash2) print('Size of the empty folder ' + keySuffixWithSlash2 + ' is ' + str(dict(resp.header).get('content-length')))
def obs_uplodfile(ak, sk, server, bucketname, localdir, obsdir_parent): ''' :param ak: obs的Access Key :param sk: obs的Secret Key :param server: obs的endpoint地址 :param bucketname: obs的名字 :param localdir: 需要上传到obs本地目录 :param obsdir_parent: 希望上传到obs后的目录名字 比如这里设置成mongodb_backup最终的备份目录为: mongodb_backup/2019-07/2019-07-08/ :return: ''' from obs import ObsClient import os import time # 创建ObsClient实例 obsClient = ObsClient(access_key_id=ak, secret_access_key=sk, server=server) DATE = time.strftime("%Y%m%d") MONTH = time.strftime("%Y%m") # 本地需要上传的目录 确保这个目录下都是文件 子目录不支持 localdir = localdir # bucketname bucketname = bucketname Logger().log('开始上传到 %s OBS ...' % bucketname) # 设置分段上传时的最大并发数 taskNum = 5 # 设置分段大小为10MB partSize = 10 * 1024 * 1024 # 开启断点续传模式 enableCheckpoint = True # obs上存放数据的文件夹 obsdir_parent = obsdir_parent obsdir = obsdir_parent + '/' + MONTH + '/' + DATE # mongodb_backup/2019-07/2019-07-08/ # 规范整理文件名和文件绝对路径 filename_list = [] abspath_list = [] for dir, _, file in os.walk(localdir): # 变量解压 filename_list = file for f in file: abspath = dir + "/" + f abspath_list.append(abspath) filename_abspath_list = zip(filename_list, abspath_list) for t in filename_abspath_list: # 设置待上传的本地文件,需要指定到具体的文件名 uploadFile = t[1] objectName = t[0] # 进行断点续传上传 try: # 创建obs存放数据的文件夹 cdir = obsClient.putContent(bucketname, obsdir, content=None) obs_filepath = os.path.join(obsdir, objectName) resp = obsClient.uploadFile(bucketname, obs_filepath, uploadFile, partSize, taskNum, enableCheckpoint) Logger().log('%s开始上传' % objectName) if resp.status < 300: Logger().log('%s上传成功\n' % objectName) # content='status:', resp['status'], 'filename:', resp['body']['key'] else: Logger().log('%s上传失败' % objectName, mode=False) #content= 'errorCode:', resp.errorCode, 'errorMessage:', resp.errorMessage # 操作失败时可再次调用断点续传上传接口进行重新上传 except Exception as e: Logger().log('%s上传失败,发生错误%s' % (objectName, e), mode=False)
SK = '*** Provide your Secret Key ***' server = 'https://your-endpoint' bucketName = 'my-obs-cold-bucket-demo' objectKey = 'my-obs-cold-object-key-demo' # Constructs a obs client instance with your account for accessing OBS obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) # Create a cold bucket print('Create a new cold bucket for demo\n') obsClient.createBucket(bucketName, CreateBucketHeader(storageClass=StorageClass.COLD)) # Create a cold object print('Create a new cold object for demo\n') obsClient.putContent(bucketName, objectKey, 'Hello OBS') # Restore the cold object print('Restore the cold object') obsClient.restoreObject(bucketName, objectKey, 1, tier=RestoreTier.EXPEDITED) # Wait 6 minute to get the object time.sleep(6 * 60) # Get the cold object status print('Get the cold object status') resp = obsClient.restoreObject(bucketName, objectKey, 1, tier=RestoreTier.EXPEDITED) print('\tstatus code ' + str(resp.status))