def upload_to_obs(model_local_path):
TestObs = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
bucketName = obs_path.split("/",1)[0]
resultFileName = obs_path.split("/",1)[1] + OBS_MODEL_DIR
TestObs.putFile(bucketName, resultFileName, file_path=model_local_path)
return 0
async def upload_file(objectKey, FilePath):
# Constructs a obs client instance with your account for accessing OBS
obsClient = ObsClient(access_key_id=AK,
secret_access_key=SK,
server=server)
sampleFilePath = 'testUpload.txt'
# Upload an object to your bucket
print('Uploading a new object to OBS from a file\n')
obsClient.putFile(bucketName, objectKey, FilePath)
# asyncio.run(upload_file())
def upload_to_obs():
# get the bucket name and OBS model path
obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
bucket_name = obs_path.split("/", 1)[0]
work_metric = obs_path.split("/", 1)[1] + '/'
work_model = obs_path.split("/", 1)[1] + '/model/'
# build bucket path
model_file = work_model + MODEL_NAME
config_file = work_model + CONFIG_NAME
metric_file = work_metric + METRIC_NAME
customize_service_file = work_model + CUSTOMIZE_SERVICE
obs_client.putContent(bucket_name, work_model, content=None)
# upload config file to OBS
print_title("upload config to obs !")
obs_client.putFile(bucket_name, config_file, file_path=LOCAL_CONFIG_PATH)
# upload custom service file to OBS
print_title("upload customize_service to obs !")
obs_client.putFile(bucket_name,
customize_service_file,
file_path=LOCAL_CUSTOMIZE_SERVICE_PATH)
# upload metric file to OBS
print_title("upload metric to obs !")
obs_client.putFile(bucket_name, metric_file, file_path=LOCAL_METRIC_PATH)
# upload model to OBS
print_title("upload model to obs !")
obs_client.putFile(bucket_name, model_file, file_path=LOCAL_MODEL_PATH)
return 0
def upload_obs(local_path, file_name):
obsClient = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
filemodel = model_path + file_name
bucketName = obs_path.split("/", 1)[0]
resp = obsClient.putFile(bucketName, filemodel, file_path=local_path)
print(resp)
return 0
async def upload_file(objectKey, FilePath):
# AK = 'N02JFWTL4SEDYUQSWNLF'
# SK = '9x5ZJ1HoXkI4L9NuOBTdsxQ2eFDh3txBohqGonT8'
# server = 'https://obs.ap-southeast-2.myhuaweicloud.com'
# bucketName = 'obs-mapper'
AK = 'IJKOCBW6LYDVI6Y1WW0Q'
SK = '7SCcFD6ROXwCAN1B5ios2tapwhvsStZMG1qxrOxy'
server = 'https://obs.ap-southeast-2.myhuaweicloud.com'
bucketName = 'obs-mapper-cie'
# Constructs a obs client instance with your account for accessing OBS
obsClient = ObsClient(access_key_id=AK,
secret_access_key=SK,
server=server)
# Upload an object to your bucket
print('Uploading a new object to OBS from a file\n')
obsClient.putFile(bucketName, objectKey, FilePath)
# asyncio.run(upload_file())
def upload_to_obs():
obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
bucket_name = obs_path.split("/", 1)[0]
work_metric = obs_path.split("/", 1)[1] + '/'
model_dir = obs_path.split("/", 1)[1] + '/model/'
model_file = model_dir + MODEL_NAME
config_file = model_dir + CONFIG_NAME
metric_file = work_metric + METRIC_NAME
# upload model to OBS
print_title("upload model to obs !")
obs_client.putFile(bucket_name,
model_file,
file_path=LOCAL_MODEL_PATH + MODEL_NAME)
# upload config file to OBS
print_title("upload config to obs !")
obs_client.putFile(bucket_name, config_file, file_path=LOCAL_CONFIG_PATH)
# upload metric file to OBS
print_title("upload metric to obs !")
obs_client.putFile(bucket_name, metric_file, file_path=LOCAL_METRIC_PATH)
return 0
def upload_to_obs():
obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
try:
bucket_name = obs_path.split("/", 1)[0]
work_metric = obs_path.split("/", 1)[1] + '/'
work_model = obs_path.split("/", 1)[1] + '/model/'
model_file = work_model + MODEL_NAME
config_file = work_model + CONFIG_NAME
metric_file = work_metric + METRIC_NAME
customize_service_file = work_model + CUSTOMIZE_SERVICE
obs_client.putContent(bucket_name, work_model, content=None)
# upload metric file to OBS
print_title("upload model to obs !")
obs_client.putFile(bucket_name, model_file, file_path=LOCAL_MODEL_PATH)
# upload config file to OBS
print_title("upload config to obs !")
obs_client.putFile(bucket_name,
config_file,
file_path=LOCAL_CONFIG_PATH)
# upload metric file to OBS
print_title("upload metric to obs !")
obs_client.putFile(bucket_name,
metric_file,
file_path=LOCAL_METRIC_PATH)
# upload custom service file to OBS
if os.path.exists(LOCAL_CUSTOMIZE_SERVICE_PATH):
print_title("upload customize_service to obs !")
obs_client.putFile(bucket_name,
customize_service_file,
file_path=LOCAL_CUSTOMIZE_SERVICE_PATH)
else:
print("user own customize_service.py not exists")
finally:
obs_client.close()
return 0
def upload_to_obs():
TestObs = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
bucketName = obs_path.split("/",1)[0]
workmetric = obs_path.split("/",1)[1] + '/'
workmodel = obs_path.split("/",1)[1] + '/model/'
#workconfig = obs_path.split("/",1)[1] + '/config/'
filemodel = workmodel + MODEL_NAME
fileconfig = workmodel + CONFIG_NAME
filemetric = workmetric + METRIC_NAME
#resultFileName = obs_path.split("/",1)[1] + '/model/xgboost.m'
#configName = obs_path.split("/",1)[1] + '/config/config.json'
TestObs.putContent(bucketName, workmodel, content=None)
#TestObs.putContent(bucketName, workconfig, content=None)
print_title("upload model to obs !")
TestObs.putFile(bucketName, filemodel, file_path=LOCAL_MODEL_DIR)
print_title("upload config to obs !")
TestObs.putFile(bucketName, fileconfig, file_path=LOCAL_CONFIG_DIR)
print_title("upload metric to obs !")
TestObs.putFile(bucketName, filemetric, file_path=LOCAL_METRIC_DIR)
return 0
print('errorCode:', resp.errorCode)
print('errorMessage:', resp.errorMessage)
raise Exception('创建桶错误')
#打开桶版本支持
resp = obsClient.setBucketVersioning(bucketName, 'Enabled')
if resp.status < 300:
print('requestId:', resp.requestId)
else:
print('errorCode:', resp.errorCode)
print('errorMessage:', resp.errorMessage)
raise Exception('打开桶多版本支持错误')
#上传训练数据到OBS
for upfile_name in upload_file_names:
resp = obsClient.putFile(bucketName, upfile_name, file_path=extranct_dir+'new'+upfile_name)
if resp.status < 300:
print('requestId:', resp.requestId)
else:
print('errorCode:', resp.errorCode)
print('errorMessage:', resp.errorMessage)
raise Exception(upfile_name+'上传到OBS错误')
else:
print("所有文件上传完毕")
print("可以开始进行预测分析")
print(bucketName+"obs.cn-north-1.myhwclouds.com")
except Exception as err:
print(err)
import traceback
f.write(str(uuid.uuid1()) + '\n')
f.write(str(uuid.uuid4()) + '\n')
return sampleFilePath
# Constructs a obs client instance with your account for accessing OBS
obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server)
# Create bucket
print('Create a new bucket for demo\n')
obsClient.createBucket(bucketName)
sampleFilePath = '/temp/test.txt'
# Upload an object to your bucket
print('Uploading a new object to OBS from a file\n')
obsClient.putFile(bucketName, objectKey, createSampleFile(sampleFilePath))
print('Downloading an object as a socket stream\n')
# Download the object as a socket stream and display it directly
resp = obsClient.getObject(bucketName, objectKey, downloadPath=None)
if resp.status < 300:
response = resp.body.response
chunk_size = 65536
if response is not None:
while True:
chunk = response.read(chunk_size)
if not chunk:
break
print(chunk)
response.close()
obs_client.putContent(bucket_name, base_folder)
resp = obs_client.getObjectMetadata(bucket_name, base_folder)
print('Size of the empty folder ' + base_folder + ' is ' +
str(dict(resp.header).get('content-length')))
code_folder = f"{base_folder}code/"
obs_client.putContent(bucket_name, code_folder)
data_folder = f"{base_folder}data/"
obs_client.putContent(bucket_name, data_folder)
model_folder = f"{base_folder}model/"
obs_client.putContent(bucket_name, model_folder)
# upload script and data to OBS
train_file = code_folder + "train.py"
obs_client.putFile(bucket_name, train_file,
"03-detect-annotation-anomalies.py")
data_file = data_folder + "kindness.csv"
obs_client.putFile(bucket_name, data_file, "data/kindness.csv")
# get resources specifications
resources = get_resources(project_id, token)["specs"]
for resource in resources:
if resource["spec_code"] == "modelarts.vm.cpu.2u":
spec_id = resource["spec_id"]
worker_server_num = resource["max_num"]
# get engines specifications
engines = get_engines(project_id, token)["engines"]
for engine in engines:
if engine["engine_name"] == "XGBoost-Sklearn":
def main(argv):
pylab.rcParams['figure.figsize'] = (8.0, 10.0)
dataDir = '/Users/xubo/Downloads'
dataType = 'train2017'
annFile = '{}/annotations/instances_{}.json'.format(dataDir, dataType)
# initialize COCO api for instance annotations
coco = COCO(annFile)
# display COCO categories and supercategories
cats = coco.loadCats(coco.getCatIds())
nms = [cat['name'] for cat in cats]
print('COCO categories: \n{}\n'.format(' '.join(nms)))
nms = set([cat['supercategory'] for cat in cats])
print('COCO supercategories: \n{}'.format(' '.join(nms)))
catIds = [10];
imgIds = coco.getImgIds(catIds=catIds);
print(imgIds.__len__())
access_key = argv[1]
secret_key = argv[2]
end_point = argv[3]
obs_client = ObsClient(
access_key_id=access_key,
secret_access_key=secret_key,
server=end_point,
long_conn_mode=True
)
prefix_s3 = "s3://"
prefix_s3_upper = "S3://"
separator = "/"
def parser_path(path):
"""
parser the path and return bucket_name and file_name
"""
base_url = str(path)[len(prefix_s3):] or str(path)[len(prefix_s3_upper):]
split_array = base_url.split(separator)
bucket_name = split_array[0]
file_name = separator.join(split_array[1:])
return bucket_name, file_name
imgIdsList = []
for i in range(imgIds.__len__()):
imgIdsList.append(imgIds[i])
imgs = coco.loadImgs(imgIdsList)
for i in range(imgIds.__len__()):
img = imgs[i]
print(i, end="\t")
print(img['file_name']);
path = "s3://modelartscarbon/traffic-light-input/" + img['file_name']
bucket_name, file = parser_path(path)
resp = obs_client.putFile(bucket_name, file,
file_path="/Users/xubo/Desktop/xubo/data/train2017/" + img['file_name'])
if resp.status < 300:
print('requestId:', resp.requestId)
else:
print('errorCode:', resp.errorCode)
print('errorMessage:', resp.errorMessage)
print(i)
# load and display image
# I = io.imread('%s/images/%s/%s'%(dataDir,dataType,img['file_name']))
# use url to load image
I = io.imread(img['coco_url'])
plt.axis('off')
plt.imshow(I)
plt.show()
# load and display instance annotations
plt.imshow(I);
plt.axis('off')
annIds = coco.getAnnIds(imgIds=img['id'], catIds=catIds, iscrowd=None)
anns = coco.loadAnns(annIds)
coco.showAnns(anns)
def backupAuditLogAction(self):
"""
# STEP 1: initial a obs client
# STEP 2: find local obssftp_audit.csv.xxx
# STEP 3: upload obssftp_audit.csv.xxx to obs bucket
# STEP 4： upload successfully, then remove local
:return:
"""
self.backupLock.acquire()
_client = None
start = time.time()
walkCost = -1
try:
list_dirs = os.walk(self._auditPath)
backFiles = []
for root, dirs, files in list_dirs:
for f in files:
if all(['%s.' % self._auditFileName in f, 'lock' not in f]):
backFiles.append(os.path.join(root, f))
if len(backFiles) < 24:
log.log(logging.DEBUG, 'Do not backup, because of backup files number [%d] little than 24.',
len(backFiles))
return
walkCost = int((time.time() - start) * 1000)
_client = ObsClient(access_key_id=self._ak, secret_access_key=self._sk, server=self._endpoint,
path_style=True)
_client.initLog(LogConf(OBS_SDK_LOG_CONFIG))
for fileAbsPath in backFiles:
log.info('Start to upload audit file [%s] to bucket [%s].', fileAbsPath, self._bucket)
filename = fileAbsPath.split('/')[-1]
dayPath = None
try:
dayPath = str(filename[len(self._auditFileName) + 1:]).split('_')[0]
except Exception as e:
log.log(logging.ERROR, 'Parse day path failed. error message [%s].', str(e))
key = ('%s/%s' % (dayPath, filename)) if dayPath else filename
try:
uploadResp = _client.putFile(self._bucket, key, file_path=fileAbsPath)
if uploadResp.status < 300:
cost = int((time.time() - start) * 1000)
log.info('Upload audit file [%s] successfully, upload cost [%d] - walk cost [%d] - %s',
fileAbsPath, cost, walkCost, self.makeResponseMessage(uploadResp))
os.remove(fileAbsPath)
continue
cost = int((time.time() - start) * 1000)
log.info(
'Upload audit file [%s] to bucket [%s] of key [%s] failed. upload cost [%d] - walk cost [%d] - %s',
fileAbsPath, key, self._bucket, cost, walkCost, self.makeErrorMessage(uploadResp))
except Exception as e:
cost = int((time.time() - start) * 1000)
log.log(
'Upload audit file [%s] to bucket [%s] of key [%s] failed. error message [%s] - upload cost [%d] - walk cost [%d]',
fileAbsPath, key, self._bucket, str(e), cost, walkCost, traceback.format_exc())
except Exception as e:
cost = int((time.time() - start) * 1000)
log.log(logging.ERROR,
'Backup audit log to bucket [%s] failed. error message [%s] - upload cost [%d] - walk cost [%d] - %s.',
self._bucket, str(e), cost, walkCost, traceback.format_exc())
pass
finally:
if _client:
_client.close()
self.backupLock.release()
if __name__ == '__main__':
# Constructs a obs client instance with your account for accessing OBS
obsClient = ObsClient(access_key_id=AK,
secret_access_key=SK,
server=server)
# Create bucket
print('Create a new bucket to upload file\n')
resp = obsClient.createBucket(bucketName)
if resp.status >= 300:
raise Exception('Create Bucket failed')
# Upload an object to your bucket
print('Uploading a new object to OBS from a file\n')
resp = obsClient.putFile(bucketName, objectKey, sampleFilePath)
if resp.status >= 300:
raise Exception('putFile failed')
# Get size of the object
resp = obsClient.getObjectMetadata(bucketName, objectKey)
if resp.status >= 300:
raise Exception('getObjectMetadata failed')
header = dict(resp.header)
objectSize = int(header.get('content-length'))
print('Object size ' + str(objectSize) + '\n')
# Calculate how many blocks to be divided
# 5MB
else:
print('\tPart#' + str(partNumber) + ' failed\n')
if __name__ == '__main__':
# Constructs a obs client instance with your account for accessing OBS
obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server)
# Create bucket
print('Create a new bucket for demo\n')
resp = obsClient.createBucket(bucketName)
if resp.status >= 300:
raise Exception('Create Bucket failed')
# # Upload an object to your source bucket
print('Uploading a new object to OBS from a file\n')
resp = obsClient.putFile(sourceBucketName, sourceObjectKey, sampleFilePath)
if resp.status >= 300:
raise Exception('putFile failed')
# Claim a upload id firstly
resp = obsClient.initiateMultipartUpload(bucketName, objectKey)
if resp.status >= 300:
raise Exception('initiateMultipartUpload failed')
uploadId = resp.body.uploadId
print('Claiming a new upload id ' + uploadId + '\n')
# 5MB
partSize = 5 * 1024 * 1024
resp = obsClient.getObjectMetadata(sourceBucketName, sourceObjectKey)
if resp.status >= 300:
class modelarts_handler():
def __init__(self):
self.output_url = None
def create_obs_output_dirs(self, output_url):
bucket_name = output_url[5:].split('/')[0]
sub_dir = output_url.replace(f"s3://{bucket_name}/", "", 1)
logger.debug('create obs output{} subdir:{} bucket:{}'.format(
output_url, sub_dir, bucket_name))
resp = self.obsClient.putContent(bucket_name, sub_dir, content=None)
if resp.status < 300:
logger.debug('obs put content request ok')
else:
logger.warn('errorCode:{} msg:{}'.format(resp.errorCode,
resp.errorMessage))
raise RuntimeError('failed')
def create_obs_handler(self, access_config):
# 创建 obs登录句柄
self.obsClient = ObsClient(
access_key_id=access_config.access_key,
secret_access_key=access_config.secret_access_key,
server=access_config.server)
def create_session(self, access_config):
# 如下配置针对计算中心等专有云 通用云不需要设置
if access_config.get("iam_endpoint") != "" and access_config.get("obs_endpoint") != "" \
and access_config.get("modelarts_endpoint") != "":
Session.set_endpoint(iam_endpoint=access_config.iam_endpoint, obs_endpoint=access_config.obs_endpoint, \
modelarts_endpoint=access_config.modelarts_endpoint, region_name=access_config.region_name)
# 创建modelarts句柄
self.session = Session(access_key=access_config.access_key,
secret_key=access_config.secret_access_key,
project_id=access_config.project_id,
region_name=access_config.region_name)
def print_train_instance_types(self):
algo_info = Estimator.get_train_instance_types(
modelarts_session=self.session)
print("get valid train_instance_types:{}".format(algo_info))
def stop_new_versions(self, session_config):
base_job_list_info = Estimator.get_job_list(
modelarts_session=self.session,
per_page=10,
page=1,
order="asc",
search_content=session_config.job_name)
if base_job_list_info == None or base_job_list_info.get(
"job_total_count", 0) == 0:
print("find no match version return")
return
else:
pre_version_id = base_job_list_info["jobs"][0].get("version_id")
job_id = base_job_list_info["jobs"][0].get("job_id")
job_status = base_job_list_info["jobs"][0].get("status")
estimator = Estimator(modelarts_session=self.session,
job_id=job_id,
version_id=pre_version_id)
if JOB_STATE[job_status] == "JOBSTAT_INIT" \
or JOB_STATE[job_status] == "JOBSTAT_IMAGE_CREATING" \
or JOB_STATE[job_status] == "JOBSTAT_SUBMIT_TRYING" \
or JOB_STATE[job_status] == "JOBSTAT_DEPLOYING" \
or JOB_STATE[job_status] == "JOBSTAT_RUNNING":
status = estimator.stop_job_version()
print(
"jobname:{} jobid:{} preversionid:{} jobstatus:{} stop status:{}"
.format(session_config.job_name, job_id, pre_version_id,
JOB_STATE[job_status], status))
else:
print(
"jobname:{} jobid:{} preversionid:{} jobstatus:{} no need stop"
.format(session_config.job_name, job_id, pre_version_id,
JOB_STATE[job_status]))
return
def get_job_name_next_new_version(self, session_config):
base_job_list_info = Estimator.get_job_list(
modelarts_session=self.session,
per_page=10,
page=1,
order="asc",
search_content=session_config.job_name)
if base_job_list_info == None or base_job_list_info.get(
"job_total_count", 0) == 0:
return 1
else:
pre_version_id = base_job_list_info["jobs"][0].get("version_id")
job_id = base_job_list_info["jobs"][0].get("job_id")
estimator = Estimator(modelarts_session=self.session,
job_id=job_id,
version_id=pre_version_id)
job_info = estimator.get_job_info()
pre_version_id = job_info.get("version_name", "V0")[1:]
return int(pre_version_id) + 1
def get_obs_url_content(self, obs_url):
bucket_name = obs_url[5:].split('/')[0]
obs_sub_path = obs_url.replace(f"s3://{bucket_name}/", "", 1)
resp = self.obsClient.getObject(bucket_name,
obs_sub_path,
loadStreamInMemory=True)
if resp.status < 300:
logger.debug('request ok')
return resp.body.buffer.decode("utf-8")
else:
raise RuntimeError(
'obs get object ret:{} url:{} bucket:{} path:{}'.format(
resp.status, obs_url, bucket_name, obs_sub_path))
def update_code_to_obs(self, session_config, localpath):
# 待完善 验证
bucket_name = session_config.code_dir.split('/')[1]
sub_dir = "/".join(session_config.code_dir.strip("/").split('/')[1:])
logger.info("update code codepath:{} bucket:{} subdir:{}".format(
session_config.code_dir, bucket_name, sub_dir))
resp = self.obsClient.putFile(bucket_name, sub_dir, localpath)
# logger.debug("lcm resp:{}".format(resp))
# print("lcm resp:{}".format(resp))
# if resp.status < 300:
# logger.debug('obs put content request ok')
# else:
# logger.warn('errorCode:{} msg:{}'.format(resp.errorCode, resp.errorMessage))
# raise RuntimeError('failed')
def create_modelarts_job(self, session_config, output_url):
jobdesc = session_config.job_description_prefix + "_jobname_" + session_config.job_name + "_" + str(
session_config.train_instance_type) + "_" + str(
session_config.train_instance_count)
estimator = Estimator(
modelarts_session=self.session,
framework_type=session_config.framework_type,
framework_version=session_config.framework_version,
code_dir=session_config.code_dir,
boot_file=session_config.boot_file,
log_url=output_url[4:],
hyperparameters=session_config.hyperparameters,
output_path=output_url[4:],
pool_id=get_config_value(session_config, "pool_id"),
train_instance_type=get_config_value(session_config,
"train_instance_type"),
train_instance_count=session_config.train_instance_count,
nas_type=get_config_value(session_config, "nas_type"),
nas_share_addr=get_config_value(session_config, "nas_share_addr"),
nas_mount_path=get_config_value(session_config, "nas_mount_path"),
job_description=jobdesc,
user_command=None)
base_job_list_info = Estimator.get_job_list(
modelarts_session=self.session,
per_page=10,
page=1,
order="asc",
search_content=session_config.job_name)
if base_job_list_info == None or base_job_list_info.get(
"job_total_count", 0) == 0:
logger.debug("new create inputs:{} job_name:{}".format(
session_config.inputs, session_config.job_name))
job_instance = estimator.fit(inputs=session_config.inputs,
wait=False,
job_name=session_config.job_name)
else:
job_id = base_job_list_info["jobs"][0].get("job_id")
pre_version_id = base_job_list_info["jobs"][0].get("version_id")
logger.debug("new versions job_id:{} pre_version_id:{}".format(
job_id, pre_version_id))
job_instance = estimator.create_job_version(
job_id=job_id,
pre_version_id=pre_version_id,
inputs=session_config.inputs,
wait=False,
job_desc=jobdesc)
print("inputs:{} job_name:{} ret instance:{}".format(
session_config.inputs, session_config.job_name, job_instance))
job_info = job_instance.get_job_info()
if not job_info['is_success']:
logger.error("failed to run job on modelarts, msg %s" %
(job_info['error_msg']))
raise RuntimeError('failed')
print(
"create sucess job_id:{} resource_id:{} version_name:{} create_time:{}"
.format(job_info["job_id"], job_info["resource_id"],
job_info["version_name"], job_info["create_time"]))
return job_instance
def run_job(self, session_config, localpath):
logger.debug("session config:{}".format(session_config))
self.print_train_instance_types()
# 获取job_name的next 版本号
next_version_id = self.get_job_name_next_new_version(session_config)
# 生成输出路径
self.output_url = os.path.join(
"s3:/{}".format(session_config.out_base_url),
"V{}".format(next_version_id), "")
logger.debug("output_url:{}".format(self.output_url))
self.create_obs_output_dirs(self.output_url)
# 更新代码到obs上
self.update_code_to_obs(session_config, localpath)
job_instance = self.create_modelarts_job(session_config,
self.output_url)
wait_for_job(job_instance)
class bucket():
def __init__(self,
access_key_id=None,
secret_access_key=None,
server=None,
bucketName=None):
# 创建ObsClient实例
self.base_folder = "/"
if access_key_id == None or secret_access_key == None or server == None:
print(
"# ----- Error invalid OBS server config please check parameter of obs server"
)
exit(0)
else:
self.obsClient = ObsClient(access_key_id=access_key_id,
secret_access_key=secret_access_key,
server=server)
if bucketName != None:
self.bucketName = bucketName
self.bucketClient = self.obsClient.bucketClient(bucketName)
print(
"# ---------------------------------------------------------------------------- #"
)
print(
"# Bucket ToolKit #"
)
print(
"# ---------------------------------------------------------------------------- #"
)
print("# ----access key (AK) : ", access_key_id)
print("# ----secret key (SK): ", secret_access_key)
print("# ----server : ", server)
print("# ----bucket name : ", bucketName)
print("# ----root : ", self.base_folder)
print(
"# ---------------------------------------------------------------------------- #"
)
def getBucketMetadata(self):
print('Getting bucket metadata\n')
#resp = obsClient.getBucketMetadata(bucketName, origin='http://www.b.com', requestHeaders='Authorization1')
resp = self.bucketClient.getBucketMetadata(
origin='http://www.b.com', requestHeaders='Authorization1')
print('storageClass:', resp.body.storageClass)
print('accessContorlAllowOrigin:', resp.body.accessContorlAllowOrigin)
print('accessContorlMaxAge:', resp.body.accessContorlMaxAge)
print('accessContorlExposeHeaders:',
resp.body.accessContorlExposeHeaders)
print('accessContorlAllowMethods:',
resp.body.accessContorlAllowMethods)
print('accessContorlAllowHeaders:',
resp.body.accessContorlAllowHeaders)
print('Deleting bucket CORS\n')
resp = self.bucketClient.deleteBucketCors()
print('status' + str(resp.status))
return resp
def upload(self, remote_path, local_path):
self.obsClient.putFile(self.bucketName, remote_path, local_path)
# print("# ===== Uploading ",local_path," ===to : ",remote_path)
def download(self, key, download):
# print("# ===== Downloading ",key," === to :",download)
self.obsClient.getObject(self.bucketName, key, downloadPath=download)
def cd(self, folder_key):
self.base_folder = folder_key
print("# ===== Base Folder", self.base_folder)
def delete(self, key):
print('# ===== Deleting object ' + key + '\n')
self.obsClient.deleteObject(self.bucketName, key)
def check(self, key):
"""
The Sync will overwrite by default. We need check
"""
assert not self.obsClient.getObject(
self.bucketName, key
)["status"] < 300, "\n# ===== ERROR : \n# ===== bucket : ({bucketname})\n# ===== key : ({key}) & local upload flow try to overwrite same key".format(
bucketname=self.bucketName, key=key)
# def mkdir(self,dir):
# pass
def ls(self, show_item_count=10):
print("# ===== list ({path}): ".format(path=self.base_folder))
# resp = self.obsClient.listObjects(self.bucketName)
if self.base_folder == "/":
resp = self.obsClient.listObjects(self.bucketName)
else:
resp = self.obsClient.listObjects(self.bucketName,
self.base_folder)
keylist = []
print("# ===== object count : ", len(resp.body.contents))
for content in resp.body.contents[:show_item_count]:
keylist.append(content.key)
print(' |--- : ' + content.key + ' etag[' + content.etag + ']')
return keylist
