if resource["spec_code"] == "modelarts.vm.cpu.2u":
spec_id = resource["spec_id"]
worker_server_num = resource["max_num"]
# get engines specifications
engines = get_engines(project_id, token)["engines"]
for engine in engines:
if engine["engine_name"] == "XGBoost-Sklearn":
engine_id = engine["engine_id"]
# run training job
job = submit_job(project_id,
token,
name=f"train-with-api-{uid}",
spec_id=spec_id,
engine_id=engine_id,
worker_server_num=worker_server_num,
app_url=f"/{bucket_name}/{code_folder}",
boot_file_url=f"/{bucket_name}/{code_folder}train.py",
data_url=f"/{bucket_name}/{data_folder}",
train_url=f"/{bucket_name}/{model_folder}")
job_id = job["job_id"]
version_id = job["version_id"]
# get job details
pprint.pprint(get_job_details(project_id, token, job_id, version_id))
# download the result file
obs_client.downloadFile(bucket_name, data_folder + "evaluation.csv",
"data/evaluation.csv")
def handle(self, *args, **options):
# 从OBS查询对象
access_key_id = os.getenv('ACCESS_KEY_ID', '')
secret_access_key = os.getenv('SECRET_ACCESS_KEY', '')
endpoint = os.getenv('OBS_ENDPOINT', '')
bucketName = os.getenv('OBS_BUCKETNAME', '')
if not access_key_id or not secret_access_key or not endpoint or not bucketName:
logger.error('losing required arguments for ObsClient')
sys.exit(1)
obs_client = ObsClient(access_key_id=access_key_id,
secret_access_key=secret_access_key,
server='https://{}'.format(endpoint))
objs = obs_client.listObjects(
bucketName=bucketName)['body']['contents']
# 遍历
if len(objs) == 0:
logger.info('OBS中无对象')
return
for obj in objs:
# 获取对象的地址
object_key = obj['key']
if not object_key.endswith('.mp4'):
continue
# 获取对象的metadata
metadata = obs_client.getObjectMetadata(bucketName, object_key)
metadata_dict = {x: y for x, y in metadata['header']}
# 如果bvid不在metadata_dict中，则下载视频并上传视频至B站
if 'bvid' in metadata_dict:
logger.info('{}已在B站上传，跳过'.format(object_key))
else:
logger.info('{}尚未上传至B站，开始下载'.format(object_key))
# 从OBS下载视频到本地临时目录
videoFile = os.path.join('/tmp', os.path.basename(object_key))
imageFile = videoFile.replace('.mp4', '.png')
if os.path.exists(videoFile):
os.remove(videoFile)
if os.path.exists(imageFile):
os.remove(imageFile)
taskNum = 5
partSize = 10 * 1024 * 1024
enableCheckpoint = True
try:
# 下载视频
resp = obs_client.downloadFile(bucketName, object_key,
videoFile, partSize,
taskNum, enableCheckpoint)
if resp.status < 300:
try:
# 下载封面
img_object_key = object_key.replace('.mp4', '.png')
try:
resp2 = obs_client.downloadFile(
bucketName, img_object_key, imageFile,
partSize, taskNum, enableCheckpoint)
if resp2.status < 300:
# 将下载的视频上传至B站
topic = metadata_dict['meeting_topic']
mid = metadata_dict['meeting_id']
community = metadata_dict['community']
record_start = metadata_dict[
'record_start']
sig = metadata_dict['sig']
date = (datetime.datetime.strptime(
record_start.replace('T', ' ').replace(
'Z', ''), "%Y-%m-%d %H:%M:%S") +
datetime.timedelta(hours=8)
).strftime('%Y-%m-%d')
res = upload(topic, date, videoFile,
imageFile, mid, sig,
community)
try:
if not Record.objects.filter(
mid=mid, platform='bilibili'):
Record.objects.create(
mid=mid, platform='bilibili')
except Exception as e:
logger.error(e)
# 修改metadata
bvid = res['bvid']
agenda = metadata_dict[
'agenda'] if 'agenda' in metadata_dict else ''
record_end = metadata_dict['record_end']
download_url = metadata_dict[
'download_url']
total_size = metadata_dict['total_size']
attenders = metadata_dict['attenders']
metadata = {
"meeting_id": mid,
"meeting_topic": topic,
"community": community,
"sig": sig,
"agenda": agenda,
"record_start": record_start,
"record_end": record_end,
"download_url": download_url,
"total_size": total_size,
"attenders": attenders,
"bvid": bvid
}
try:
resp3 = obs_client.setObjectMetadata(
bucketName, object_key, metadata)
if resp3.status < 300:
logger.info(
'{}: metadata修改成功'.format(
object_key))
else:
logger.error(
'errorCode', resp3.errorCode)
logger.error(
'errorMessage',
resp3.errorMessage)
except:
logger.error(traceback.format_exc())
# 休眠30s避免上传间隔过短
time.sleep(30)
else:
logger.error('errorCode', resp2.errorCode)
logger.error('errorMessage',
resp2.errorMessage)
except Exception as e2:
logger.error(e2)
except:
logger.error(traceback.format_exc())
else:
logger.error('errorCode', resp.errorCode)
logger.error('errorMessage', resp.errorMessage)
except:
logger.error(traceback.format_exc())
