def upload_to_obs(model_local_path): TestObs = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) bucketName = obs_path.split("/",1)[0] resultFileName = obs_path.split("/",1)[1] + OBS_MODEL_DIR TestObs.putFile(bucketName, resultFileName, file_path=model_local_path) return 0
async def upload_file(objectKey, FilePath): # Constructs a obs client instance with your account for accessing OBS obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) sampleFilePath = 'testUpload.txt' # Upload an object to your bucket print('Uploading a new object to OBS from a file\n') obsClient.putFile(bucketName, objectKey, FilePath) # asyncio.run(upload_file())
def upload_to_obs(): # get the bucket name and OBS model path obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) bucket_name = obs_path.split("/", 1)[0] work_metric = obs_path.split("/", 1)[1] + '/' work_model = obs_path.split("/", 1)[1] + '/model/' # build bucket path model_file = work_model + MODEL_NAME config_file = work_model + CONFIG_NAME metric_file = work_metric + METRIC_NAME customize_service_file = work_model + CUSTOMIZE_SERVICE obs_client.putContent(bucket_name, work_model, content=None) # upload config file to OBS print_title("upload config to obs !") obs_client.putFile(bucket_name, config_file, file_path=LOCAL_CONFIG_PATH) # upload custom service file to OBS print_title("upload customize_service to obs !") obs_client.putFile(bucket_name, customize_service_file, file_path=LOCAL_CUSTOMIZE_SERVICE_PATH) # upload metric file to OBS print_title("upload metric to obs !") obs_client.putFile(bucket_name, metric_file, file_path=LOCAL_METRIC_PATH) # upload model to OBS print_title("upload model to obs !") obs_client.putFile(bucket_name, model_file, file_path=LOCAL_MODEL_PATH) return 0
def upload_obs(local_path, file_name): obsClient = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) filemodel = model_path + file_name bucketName = obs_path.split("/", 1)[0] resp = obsClient.putFile(bucketName, filemodel, file_path=local_path) print(resp) return 0
async def upload_file(objectKey, FilePath): # AK = 'N02JFWTL4SEDYUQSWNLF' # SK = '9x5ZJ1HoXkI4L9NuOBTdsxQ2eFDh3txBohqGonT8' # server = 'https://obs.ap-southeast-2.myhuaweicloud.com' # bucketName = 'obs-mapper' AK = 'IJKOCBW6LYDVI6Y1WW0Q' SK = '7SCcFD6ROXwCAN1B5ios2tapwhvsStZMG1qxrOxy' server = 'https://obs.ap-southeast-2.myhuaweicloud.com' bucketName = 'obs-mapper-cie' # Constructs a obs client instance with your account for accessing OBS obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) # Upload an object to your bucket print('Uploading a new object to OBS from a file\n') obsClient.putFile(bucketName, objectKey, FilePath) # asyncio.run(upload_file())
def upload_to_obs(): obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) bucket_name = obs_path.split("/", 1)[0] work_metric = obs_path.split("/", 1)[1] + '/' model_dir = obs_path.split("/", 1)[1] + '/model/' model_file = model_dir + MODEL_NAME config_file = model_dir + CONFIG_NAME metric_file = work_metric + METRIC_NAME # upload model to OBS print_title("upload model to obs !") obs_client.putFile(bucket_name, model_file, file_path=LOCAL_MODEL_PATH + MODEL_NAME) # upload config file to OBS print_title("upload config to obs !") obs_client.putFile(bucket_name, config_file, file_path=LOCAL_CONFIG_PATH) # upload metric file to OBS print_title("upload metric to obs !") obs_client.putFile(bucket_name, metric_file, file_path=LOCAL_METRIC_PATH) return 0
def upload_to_obs(): obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) try: bucket_name = obs_path.split("/", 1)[0] work_metric = obs_path.split("/", 1)[1] + '/' work_model = obs_path.split("/", 1)[1] + '/model/' model_file = work_model + MODEL_NAME config_file = work_model + CONFIG_NAME metric_file = work_metric + METRIC_NAME customize_service_file = work_model + CUSTOMIZE_SERVICE obs_client.putContent(bucket_name, work_model, content=None) # upload metric file to OBS print_title("upload model to obs !") obs_client.putFile(bucket_name, model_file, file_path=LOCAL_MODEL_PATH) # upload config file to OBS print_title("upload config to obs !") obs_client.putFile(bucket_name, config_file, file_path=LOCAL_CONFIG_PATH) # upload metric file to OBS print_title("upload metric to obs !") obs_client.putFile(bucket_name, metric_file, file_path=LOCAL_METRIC_PATH) # upload custom service file to OBS if os.path.exists(LOCAL_CUSTOMIZE_SERVICE_PATH): print_title("upload customize_service to obs !") obs_client.putFile(bucket_name, customize_service_file, file_path=LOCAL_CUSTOMIZE_SERVICE_PATH) else: print("user own customize_service.py not exists") finally: obs_client.close() return 0
def upload_to_obs(): TestObs = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) bucketName = obs_path.split("/",1)[0] workmetric = obs_path.split("/",1)[1] + '/' workmodel = obs_path.split("/",1)[1] + '/model/' #workconfig = obs_path.split("/",1)[1] + '/config/' filemodel = workmodel + MODEL_NAME fileconfig = workmodel + CONFIG_NAME filemetric = workmetric + METRIC_NAME #resultFileName = obs_path.split("/",1)[1] + '/model/xgboost.m' #configName = obs_path.split("/",1)[1] + '/config/config.json' TestObs.putContent(bucketName, workmodel, content=None) #TestObs.putContent(bucketName, workconfig, content=None) print_title("upload model to obs !") TestObs.putFile(bucketName, filemodel, file_path=LOCAL_MODEL_DIR) print_title("upload config to obs !") TestObs.putFile(bucketName, fileconfig, file_path=LOCAL_CONFIG_DIR) print_title("upload metric to obs !") TestObs.putFile(bucketName, filemetric, file_path=LOCAL_METRIC_DIR) return 0
print('errorCode:', resp.errorCode) print('errorMessage:', resp.errorMessage) raise Exception('创建桶错误') #打开桶版本支持 resp = obsClient.setBucketVersioning(bucketName, 'Enabled') if resp.status < 300: print('requestId:', resp.requestId) else: print('errorCode:', resp.errorCode) print('errorMessage:', resp.errorMessage) raise Exception('打开桶多版本支持错误') #上传训练数据到OBS for upfile_name in upload_file_names: resp = obsClient.putFile(bucketName, upfile_name, file_path=extranct_dir+'new'+upfile_name) if resp.status < 300: print('requestId:', resp.requestId) else: print('errorCode:', resp.errorCode) print('errorMessage:', resp.errorMessage) raise Exception(upfile_name+'上传到OBS错误') else: print("所有文件上传完毕") print("可以开始进行预测分析") print(bucketName+"obs.cn-north-1.myhwclouds.com") except Exception as err: print(err) import traceback
f.write(str(uuid.uuid1()) + '\n') f.write(str(uuid.uuid4()) + '\n') return sampleFilePath # Constructs a obs client instance with your account for accessing OBS obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) # Create bucket print('Create a new bucket for demo\n') obsClient.createBucket(bucketName) sampleFilePath = '/temp/test.txt' # Upload an object to your bucket print('Uploading a new object to OBS from a file\n') obsClient.putFile(bucketName, objectKey, createSampleFile(sampleFilePath)) print('Downloading an object as a socket stream\n') # Download the object as a socket stream and display it directly resp = obsClient.getObject(bucketName, objectKey, downloadPath=None) if resp.status < 300: response = resp.body.response chunk_size = 65536 if response is not None: while True: chunk = response.read(chunk_size) if not chunk: break print(chunk) response.close()
obs_client.putContent(bucket_name, base_folder) resp = obs_client.getObjectMetadata(bucket_name, base_folder) print('Size of the empty folder ' + base_folder + ' is ' + str(dict(resp.header).get('content-length'))) code_folder = f"{base_folder}code/" obs_client.putContent(bucket_name, code_folder) data_folder = f"{base_folder}data/" obs_client.putContent(bucket_name, data_folder) model_folder = f"{base_folder}model/" obs_client.putContent(bucket_name, model_folder) # upload script and data to OBS train_file = code_folder + "train.py" obs_client.putFile(bucket_name, train_file, "03-detect-annotation-anomalies.py") data_file = data_folder + "kindness.csv" obs_client.putFile(bucket_name, data_file, "data/kindness.csv") # get resources specifications resources = get_resources(project_id, token)["specs"] for resource in resources: if resource["spec_code"] == "modelarts.vm.cpu.2u": spec_id = resource["spec_id"] worker_server_num = resource["max_num"] # get engines specifications engines = get_engines(project_id, token)["engines"] for engine in engines: if engine["engine_name"] == "XGBoost-Sklearn":
def main(argv): pylab.rcParams['figure.figsize'] = (8.0, 10.0) dataDir = '/Users/xubo/Downloads' dataType = 'train2017' annFile = '{}/annotations/instances_{}.json'.format(dataDir, dataType) # initialize COCO api for instance annotations coco = COCO(annFile) # display COCO categories and supercategories cats = coco.loadCats(coco.getCatIds()) nms = [cat['name'] for cat in cats] print('COCO categories: \n{}\n'.format(' '.join(nms))) nms = set([cat['supercategory'] for cat in cats]) print('COCO supercategories: \n{}'.format(' '.join(nms))) catIds = [10]; imgIds = coco.getImgIds(catIds=catIds); print(imgIds.__len__()) access_key = argv[1] secret_key = argv[2] end_point = argv[3] obs_client = ObsClient( access_key_id=access_key, secret_access_key=secret_key, server=end_point, long_conn_mode=True ) prefix_s3 = "s3://" prefix_s3_upper = "S3://" separator = "/" def parser_path(path): """ parser the path and return bucket_name and file_name """ base_url = str(path)[len(prefix_s3):] or str(path)[len(prefix_s3_upper):] split_array = base_url.split(separator) bucket_name = split_array[0] file_name = separator.join(split_array[1:]) return bucket_name, file_name imgIdsList = [] for i in range(imgIds.__len__()): imgIdsList.append(imgIds[i]) imgs = coco.loadImgs(imgIdsList) for i in range(imgIds.__len__()): img = imgs[i] print(i, end="\t") print(img['file_name']); path = "s3://modelartscarbon/traffic-light-input/" + img['file_name'] bucket_name, file = parser_path(path) resp = obs_client.putFile(bucket_name, file, file_path="/Users/xubo/Desktop/xubo/data/train2017/" + img['file_name']) if resp.status < 300: print('requestId:', resp.requestId) else: print('errorCode:', resp.errorCode) print('errorMessage:', resp.errorMessage) print(i) # load and display image # I = io.imread('%s/images/%s/%s'%(dataDir,dataType,img['file_name'])) # use url to load image I = io.imread(img['coco_url']) plt.axis('off') plt.imshow(I) plt.show() # load and display instance annotations plt.imshow(I); plt.axis('off') annIds = coco.getAnnIds(imgIds=img['id'], catIds=catIds, iscrowd=None) anns = coco.loadAnns(annIds) coco.showAnns(anns)
def backupAuditLogAction(self): """ # STEP 1: initial a obs client # STEP 2: find local obssftp_audit.csv.xxx # STEP 3: upload obssftp_audit.csv.xxx to obs bucket # STEP 4: upload successfully, then remove local :return: """ self.backupLock.acquire() _client = None start = time.time() walkCost = -1 try: list_dirs = os.walk(self._auditPath) backFiles = [] for root, dirs, files in list_dirs: for f in files: if all(['%s.' % self._auditFileName in f, 'lock' not in f]): backFiles.append(os.path.join(root, f)) if len(backFiles) < 24: log.log(logging.DEBUG, 'Do not backup, because of backup files number [%d] little than 24.', len(backFiles)) return walkCost = int((time.time() - start) * 1000) _client = ObsClient(access_key_id=self._ak, secret_access_key=self._sk, server=self._endpoint, path_style=True) _client.initLog(LogConf(OBS_SDK_LOG_CONFIG)) for fileAbsPath in backFiles: log.info('Start to upload audit file [%s] to bucket [%s].', fileAbsPath, self._bucket) filename = fileAbsPath.split('/')[-1] dayPath = None try: dayPath = str(filename[len(self._auditFileName) + 1:]).split('_')[0] except Exception as e: log.log(logging.ERROR, 'Parse day path failed. error message [%s].', str(e)) key = ('%s/%s' % (dayPath, filename)) if dayPath else filename try: uploadResp = _client.putFile(self._bucket, key, file_path=fileAbsPath) if uploadResp.status < 300: cost = int((time.time() - start) * 1000) log.info('Upload audit file [%s] successfully, upload cost [%d] - walk cost [%d] - %s', fileAbsPath, cost, walkCost, self.makeResponseMessage(uploadResp)) os.remove(fileAbsPath) continue cost = int((time.time() - start) * 1000) log.info( 'Upload audit file [%s] to bucket [%s] of key [%s] failed. upload cost [%d] - walk cost [%d] - %s', fileAbsPath, key, self._bucket, cost, walkCost, self.makeErrorMessage(uploadResp)) except Exception as e: cost = int((time.time() - start) * 1000) log.log( 'Upload audit file [%s] to bucket [%s] of key [%s] failed. error message [%s] - upload cost [%d] - walk cost [%d]', fileAbsPath, key, self._bucket, str(e), cost, walkCost, traceback.format_exc()) except Exception as e: cost = int((time.time() - start) * 1000) log.log(logging.ERROR, 'Backup audit log to bucket [%s] failed. error message [%s] - upload cost [%d] - walk cost [%d] - %s.', self._bucket, str(e), cost, walkCost, traceback.format_exc()) pass finally: if _client: _client.close() self.backupLock.release()
if __name__ == '__main__': # Constructs a obs client instance with your account for accessing OBS obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) # Create bucket print('Create a new bucket to upload file\n') resp = obsClient.createBucket(bucketName) if resp.status >= 300: raise Exception('Create Bucket failed') # Upload an object to your bucket print('Uploading a new object to OBS from a file\n') resp = obsClient.putFile(bucketName, objectKey, sampleFilePath) if resp.status >= 300: raise Exception('putFile failed') # Get size of the object resp = obsClient.getObjectMetadata(bucketName, objectKey) if resp.status >= 300: raise Exception('getObjectMetadata failed') header = dict(resp.header) objectSize = int(header.get('content-length')) print('Object size ' + str(objectSize) + '\n') # Calculate how many blocks to be divided # 5MB
else: print('\tPart#' + str(partNumber) + ' failed\n') if __name__ == '__main__': # Constructs a obs client instance with your account for accessing OBS obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) # Create bucket print('Create a new bucket for demo\n') resp = obsClient.createBucket(bucketName) if resp.status >= 300: raise Exception('Create Bucket failed') # # Upload an object to your source bucket print('Uploading a new object to OBS from a file\n') resp = obsClient.putFile(sourceBucketName, sourceObjectKey, sampleFilePath) if resp.status >= 300: raise Exception('putFile failed') # Claim a upload id firstly resp = obsClient.initiateMultipartUpload(bucketName, objectKey) if resp.status >= 300: raise Exception('initiateMultipartUpload failed') uploadId = resp.body.uploadId print('Claiming a new upload id ' + uploadId + '\n') # 5MB partSize = 5 * 1024 * 1024 resp = obsClient.getObjectMetadata(sourceBucketName, sourceObjectKey) if resp.status >= 300:
class modelarts_handler(): def __init__(self): self.output_url = None def create_obs_output_dirs(self, output_url): bucket_name = output_url[5:].split('/')[0] sub_dir = output_url.replace(f"s3://{bucket_name}/", "", 1) logger.debug('create obs output{} subdir:{} bucket:{}'.format( output_url, sub_dir, bucket_name)) resp = self.obsClient.putContent(bucket_name, sub_dir, content=None) if resp.status < 300: logger.debug('obs put content request ok') else: logger.warn('errorCode:{} msg:{}'.format(resp.errorCode, resp.errorMessage)) raise RuntimeError('failed') def create_obs_handler(self, access_config): # 创建 obs登录句柄 self.obsClient = ObsClient( access_key_id=access_config.access_key, secret_access_key=access_config.secret_access_key, server=access_config.server) def create_session(self, access_config): # 如下配置针对计算中心等专有云 通用云不需要设置 if access_config.get("iam_endpoint") != "" and access_config.get("obs_endpoint") != "" \ and access_config.get("modelarts_endpoint") != "": Session.set_endpoint(iam_endpoint=access_config.iam_endpoint, obs_endpoint=access_config.obs_endpoint, \ modelarts_endpoint=access_config.modelarts_endpoint, region_name=access_config.region_name) # 创建modelarts句柄 self.session = Session(access_key=access_config.access_key, secret_key=access_config.secret_access_key, project_id=access_config.project_id, region_name=access_config.region_name) def print_train_instance_types(self): algo_info = Estimator.get_train_instance_types( modelarts_session=self.session) print("get valid train_instance_types:{}".format(algo_info)) def stop_new_versions(self, session_config): base_job_list_info = Estimator.get_job_list( modelarts_session=self.session, per_page=10, page=1, order="asc", search_content=session_config.job_name) if base_job_list_info == None or base_job_list_info.get( "job_total_count", 0) == 0: print("find no match version return") return else: pre_version_id = base_job_list_info["jobs"][0].get("version_id") job_id = base_job_list_info["jobs"][0].get("job_id") job_status = base_job_list_info["jobs"][0].get("status") estimator = Estimator(modelarts_session=self.session, job_id=job_id, version_id=pre_version_id) if JOB_STATE[job_status] == "JOBSTAT_INIT" \ or JOB_STATE[job_status] == "JOBSTAT_IMAGE_CREATING" \ or JOB_STATE[job_status] == "JOBSTAT_SUBMIT_TRYING" \ or JOB_STATE[job_status] == "JOBSTAT_DEPLOYING" \ or JOB_STATE[job_status] == "JOBSTAT_RUNNING": status = estimator.stop_job_version() print( "jobname:{} jobid:{} preversionid:{} jobstatus:{} stop status:{}" .format(session_config.job_name, job_id, pre_version_id, JOB_STATE[job_status], status)) else: print( "jobname:{} jobid:{} preversionid:{} jobstatus:{} no need stop" .format(session_config.job_name, job_id, pre_version_id, JOB_STATE[job_status])) return def get_job_name_next_new_version(self, session_config): base_job_list_info = Estimator.get_job_list( modelarts_session=self.session, per_page=10, page=1, order="asc", search_content=session_config.job_name) if base_job_list_info == None or base_job_list_info.get( "job_total_count", 0) == 0: return 1 else: pre_version_id = base_job_list_info["jobs"][0].get("version_id") job_id = base_job_list_info["jobs"][0].get("job_id") estimator = Estimator(modelarts_session=self.session, job_id=job_id, version_id=pre_version_id) job_info = estimator.get_job_info() pre_version_id = job_info.get("version_name", "V0")[1:] return int(pre_version_id) + 1 def get_obs_url_content(self, obs_url): bucket_name = obs_url[5:].split('/')[0] obs_sub_path = obs_url.replace(f"s3://{bucket_name}/", "", 1) resp = self.obsClient.getObject(bucket_name, obs_sub_path, loadStreamInMemory=True) if resp.status < 300: logger.debug('request ok') return resp.body.buffer.decode("utf-8") else: raise RuntimeError( 'obs get object ret:{} url:{} bucket:{} path:{}'.format( resp.status, obs_url, bucket_name, obs_sub_path)) def update_code_to_obs(self, session_config, localpath): # 待完善 验证 bucket_name = session_config.code_dir.split('/')[1] sub_dir = "/".join(session_config.code_dir.strip("/").split('/')[1:]) logger.info("update code codepath:{} bucket:{} subdir:{}".format( session_config.code_dir, bucket_name, sub_dir)) resp = self.obsClient.putFile(bucket_name, sub_dir, localpath) # logger.debug("lcm resp:{}".format(resp)) # print("lcm resp:{}".format(resp)) # if resp.status < 300: # logger.debug('obs put content request ok') # else: # logger.warn('errorCode:{} msg:{}'.format(resp.errorCode, resp.errorMessage)) # raise RuntimeError('failed') def create_modelarts_job(self, session_config, output_url): jobdesc = session_config.job_description_prefix + "_jobname_" + session_config.job_name + "_" + str( session_config.train_instance_type) + "_" + str( session_config.train_instance_count) estimator = Estimator( modelarts_session=self.session, framework_type=session_config.framework_type, framework_version=session_config.framework_version, code_dir=session_config.code_dir, boot_file=session_config.boot_file, log_url=output_url[4:], hyperparameters=session_config.hyperparameters, output_path=output_url[4:], pool_id=get_config_value(session_config, "pool_id"), train_instance_type=get_config_value(session_config, "train_instance_type"), train_instance_count=session_config.train_instance_count, nas_type=get_config_value(session_config, "nas_type"), nas_share_addr=get_config_value(session_config, "nas_share_addr"), nas_mount_path=get_config_value(session_config, "nas_mount_path"), job_description=jobdesc, user_command=None) base_job_list_info = Estimator.get_job_list( modelarts_session=self.session, per_page=10, page=1, order="asc", search_content=session_config.job_name) if base_job_list_info == None or base_job_list_info.get( "job_total_count", 0) == 0: logger.debug("new create inputs:{} job_name:{}".format( session_config.inputs, session_config.job_name)) job_instance = estimator.fit(inputs=session_config.inputs, wait=False, job_name=session_config.job_name) else: job_id = base_job_list_info["jobs"][0].get("job_id") pre_version_id = base_job_list_info["jobs"][0].get("version_id") logger.debug("new versions job_id:{} pre_version_id:{}".format( job_id, pre_version_id)) job_instance = estimator.create_job_version( job_id=job_id, pre_version_id=pre_version_id, inputs=session_config.inputs, wait=False, job_desc=jobdesc) print("inputs:{} job_name:{} ret instance:{}".format( session_config.inputs, session_config.job_name, job_instance)) job_info = job_instance.get_job_info() if not job_info['is_success']: logger.error("failed to run job on modelarts, msg %s" % (job_info['error_msg'])) raise RuntimeError('failed') print( "create sucess job_id:{} resource_id:{} version_name:{} create_time:{}" .format(job_info["job_id"], job_info["resource_id"], job_info["version_name"], job_info["create_time"])) return job_instance def run_job(self, session_config, localpath): logger.debug("session config:{}".format(session_config)) self.print_train_instance_types() # 获取job_name的next 版本号 next_version_id = self.get_job_name_next_new_version(session_config) # 生成输出路径 self.output_url = os.path.join( "s3:/{}".format(session_config.out_base_url), "V{}".format(next_version_id), "") logger.debug("output_url:{}".format(self.output_url)) self.create_obs_output_dirs(self.output_url) # 更新代码到obs上 self.update_code_to_obs(session_config, localpath) job_instance = self.create_modelarts_job(session_config, self.output_url) wait_for_job(job_instance)
class bucket(): def __init__(self, access_key_id=None, secret_access_key=None, server=None, bucketName=None): # 创建ObsClient实例 self.base_folder = "/" if access_key_id == None or secret_access_key == None or server == None: print( "# ----- Error invalid OBS server config please check parameter of obs server" ) exit(0) else: self.obsClient = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server=server) if bucketName != None: self.bucketName = bucketName self.bucketClient = self.obsClient.bucketClient(bucketName) print( "# ---------------------------------------------------------------------------- #" ) print( "# Bucket ToolKit #" ) print( "# ---------------------------------------------------------------------------- #" ) print("# ----access key (AK) : ", access_key_id) print("# ----secret key (SK): ", secret_access_key) print("# ----server : ", server) print("# ----bucket name : ", bucketName) print("# ----root : ", self.base_folder) print( "# ---------------------------------------------------------------------------- #" ) def getBucketMetadata(self): print('Getting bucket metadata\n') #resp = obsClient.getBucketMetadata(bucketName, origin='http://www.b.com', requestHeaders='Authorization1') resp = self.bucketClient.getBucketMetadata( origin='http://www.b.com', requestHeaders='Authorization1') print('storageClass:', resp.body.storageClass) print('accessContorlAllowOrigin:', resp.body.accessContorlAllowOrigin) print('accessContorlMaxAge:', resp.body.accessContorlMaxAge) print('accessContorlExposeHeaders:', resp.body.accessContorlExposeHeaders) print('accessContorlAllowMethods:', resp.body.accessContorlAllowMethods) print('accessContorlAllowHeaders:', resp.body.accessContorlAllowHeaders) print('Deleting bucket CORS\n') resp = self.bucketClient.deleteBucketCors() print('status' + str(resp.status)) return resp def upload(self, remote_path, local_path): self.obsClient.putFile(self.bucketName, remote_path, local_path) # print("# ===== Uploading ",local_path," ===to : ",remote_path) def download(self, key, download): # print("# ===== Downloading ",key," === to :",download) self.obsClient.getObject(self.bucketName, key, downloadPath=download) def cd(self, folder_key): self.base_folder = folder_key print("# ===== Base Folder", self.base_folder) def delete(self, key): print('# ===== Deleting object ' + key + '\n') self.obsClient.deleteObject(self.bucketName, key) def check(self, key): """ The Sync will overwrite by default. We need check """ assert not self.obsClient.getObject( self.bucketName, key )["status"] < 300, "\n# ===== ERROR : \n# ===== bucket : ({bucketname})\n# ===== key : ({key}) & local upload flow try to overwrite same key".format( bucketname=self.bucketName, key=key) # def mkdir(self,dir): # pass def ls(self, show_item_count=10): print("# ===== list ({path}): ".format(path=self.base_folder)) # resp = self.obsClient.listObjects(self.bucketName) if self.base_folder == "/": resp = self.obsClient.listObjects(self.bucketName) else: resp = self.obsClient.listObjects(self.bucketName, self.base_folder) keylist = [] print("# ===== object count : ", len(resp.body.contents)) for content in resp.body.contents[:show_item_count]: keylist.append(content.key) print(' |--- : ' + content.key + ' etag[' + content.etag + ']') return keylist