def handle(self, *args, **options):
uid = int(os.getenv('BILI_UID', ''))
if not uid:
logger.error('uid is required')
sys.exit(1)
access_key_id = os.getenv('ACCESS_KEY_ID', '')
secret_access_key = os.getenv('SECRET_ACCESS_KEY', '')
endpoint = os.getenv('OBS_ENDPOINT', '')
bucketName = os.getenv('OBS_BUCKETNAME', '')
if not access_key_id or not secret_access_key or not endpoint or not bucketName:
logger.error('losing required arguments for ObsClient')
sys.exit(1)
# 获取OBS openeuler/meetup/下的MP4列表
obs_client = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key,
server='https://{}'.format(endpoint))
objs = obs_client.listObjects(bucketName=bucketName)['body']['contents']
meetup_videos = []
for obj in objs:
if obj['key'].startswith('openeuler/meetup/') and obj['key'].endswith('.mp4'):
meetup_videos.append(obj['key'])
if len(meetup_videos) == 0:
logger.info('no meetup videos in OBS')
return
logger.info('meetup_videos: {}'.format(meetup_videos))
videos = get_videos_g(uid)
bvs = [x['bvid'] for x in videos]
遍历meetup_videos，若obj的metadata无bvid，则下载上传B站
for video in meetup_videos:
metadata = obs_client.getObjectMetadata(bucketName, video)
metadata_dict = {x: y for x, y in metadata['header']}
topic = os.path.basename(video)[:-4]
activity_id = int(video.split('/')[2])
replay_url = 'https://{}.{}/{}'.format(bucketName, endpoint, video)
Activity.objects.filter(id=activity_id).update(replay_url=replay_url)
logger.info('meetup回放视频同步小程序，回放链接: {}'.format(replay_url))
if 'bvid' not in metadata_dict.keys():
# 下载视频
logger.info('开始下载视频')
videoFile = download_video(obs_client, bucketName, video)
logger.info('视频已下载: {}'.format(videoFile))
# 生成封面
logger.info('开始生成封面')
logger.info('topic: {}'.format(topic))
imageFile = generate_cover(topic)
logger.info('封面已生成: {}'.format(imageFile))
# 上传B站
logger.info('开始上传B站')
bvid = upload_to_bilibili(videoFile, imageFile, topic)
logger.info('B站上传成功，bvid: {}'.format(bvid))
# 更新metadata
metadata = {
'bvid': bvid
}
r = obs_client.setObjectMetadata(bucketName, video, metadata)
if r.status < 300:
logger.info('更新metadata')
else:
logger.error(r.errorCode, r.errorMessage)
def handle(self, *args, **options):
uid = int(os.getenv('BILI_UID', ''))
if not uid:
logger.error('uid is required')
sys.exit(1)
videos = get_videos_g(uid=uid)
# 所有过审视频的bvid集合
bvs = [x['bvid'] for x in videos]
logger.info('所有B站过审视频的bvid: {}'.format(bvs))
logger.info('B站过审视频数: {}'.format(len(bvs)))
access_key_id = os.getenv('ACCESS_KEY_ID', '')
secret_access_key = os.getenv('SECRET_ACCESS_KEY', '')
endpoint = os.getenv('OBS_ENDPOINT', '')
bucketName = os.getenv('OBS_BUCKETNAME', '')
if not access_key_id or not secret_access_key or not endpoint or not bucketName:
logger.error('losing required arguments for ObsClient')
sys.exit(1)
obs_client = ObsClient(access_key_id=access_key_id,
secret_access_key=secret_access_key,
server='https://{}'.format(endpoint))
bili_mids = [
int(x.mid) for x in Record.objects.filter(platform='bilibili',
url__isnull=True)
]
logger.info('所有还未上传B站的会议的mid: {}'.format(bili_mids))
all_bili_mids = [
int(x.mid) for x in Record.objects.filter(platform='bilibili')
]
for mid in all_bili_mids:
obs_record = Record.objects.get(mid=mid, platform='obs')
url = obs_record.url
object_key = url.split('/', 3)[-1]
# 获取对象的metadata
metadata = obs_client.getObjectMetadata(bucketName, object_key)
metadata_dict = {x: y for x, y in metadata['header']}
if 'bvid' not in metadata_dict.keys():
logger.info('meeting {}: 未上传B站，跳过'.format(mid))
else:
logger.info('meeting {}: bvid为{}'.format(
mid, metadata_dict['bvid']))
if metadata_dict['bvid'] not in bvs:
logger.info('meetings: {}: 上传至B站，还未过审'.format(mid))
else:
bili_url = 'https://www.bilibili.com/{}'.format(
metadata_dict['bvid'])
Record.objects.filter(
mid=mid, platform='bilibili').update(url=bili_url)
logger.info('meeting {}: B站已过审，刷新播放地址'.format(mid))
# Create bucket
print('Create a new bucket for demo\n')
obsClient.createBucket(bucketName)
# Setting object mime type
headers = PutObjectHeader(contentType='text/plain')
# Setting self-defined metadata
metadata = {'meta1': 'value1', 'meta2': 'value2'}
resp = obsClient.putContent(bucketName,
objectKey,
'Hello OBS',
metadata=metadata,
headers=headers)
if resp.status < 300:
print('Create object ' + objectKey + ' successfully!\n')
else:
print('common msg:status:', resp.status, ',errorCode:', resp.errorCode,
',errorMessage:', resp.errorMessage)
# Get object metadata
resp = obsClient.getObjectMetadata(bucketName, objectKey)
header = dict(resp.header)
print('\tContentType:' + header.get('content-type'))
print('\tmeta1:' + header.get('meta1'))
print('\tmeta2:' + header.get('meta2'))
obsClient.deleteObject(bucketName, objectKey)
# get ak/sk credentials
credentials = create_temporary_ak_sk(token)
AK = credentials["access"]
SK = credentials["secret"]
# Constructs a obs client instance with your account for accessing OBS
obs_client = ObsClient(access_key_id=AK,
secret_access_key=SK,
server=server)
bucket_client = obs_client.bucketClient(bucket_name)
# TODO create OBS folders (not working)
base_folder = f"{uid}/"
obs_client.putContent(bucket_name, base_folder)
resp = obs_client.getObjectMetadata(bucket_name, base_folder)
print('Size of the empty folder ' + base_folder + ' is ' +
str(dict(resp.header).get('content-length')))
code_folder = f"{base_folder}code/"
obs_client.putContent(bucket_name, code_folder)
data_folder = f"{base_folder}data/"
obs_client.putContent(bucket_name, data_folder)
model_folder = f"{base_folder}model/"
obs_client.putContent(bucket_name, model_folder)
# upload script and data to OBS
train_file = code_folder + "train.py"
obs_client.putFile(bucket_name, train_file,
"03-detect-annotation-anomalies.py")
from __future__ import print_function
from obs import ObsClient
AK = '*** Provide your Access Key ***'
SK = '*** Provide your Secret Key ***'
server = 'https://your-endpoint'
bucketName = 'my-obs-bucket-demo'
# Constructs a obs client instance with your account for accessing OBS
obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server)
# Create bucket
print('Create a new bucket for demo\n')
obsClient.createBucket(bucketName)
keySuffixWithSlash1 = 'MyObjectKey1/'
keySuffixWithSlash2 = 'MyObjectKey2/'
# Create two empty folder without request body, note that the key must be suffixed with a slash
obsClient.putContent(bucketName, keySuffixWithSlash1, '')
print('Creating an empty folder ' + keySuffixWithSlash1)
obsClient.putContent(bucketName, keySuffixWithSlash2)
print('Creating an empty folder ' + keySuffixWithSlash2)
# Verify whether the size of the empty folder is zero
resp = obsClient.getObjectMetadata(bucketName, keySuffixWithSlash1)
print('Size of the empty folder ' + keySuffixWithSlash1 + ' is ' + str(dict(resp.header).get('content-length')))
resp = obsClient.getObjectMetadata(bucketName, keySuffixWithSlash2)
print('Size of the empty folder ' + keySuffixWithSlash2 + ' is ' + str(dict(resp.header).get('content-length')))
print('Uploading a new object to OBS from a file\n')
resp = obsClient.putFile(sourceBucketName, sourceObjectKey, sampleFilePath)
if resp.status >= 300:
raise Exception('putFile failed')
# Claim a upload id firstly
resp = obsClient.initiateMultipartUpload(bucketName, objectKey)
if resp.status >= 300:
raise Exception('initiateMultipartUpload failed')
uploadId = resp.body.uploadId
print('Claiming a new upload id ' + uploadId + '\n')
# 5MB
partSize = 5 * 1024 * 1024
resp = obsClient.getObjectMetadata(sourceBucketName, sourceObjectKey)
if resp.status >= 300:
raise Exception('getObjectMetadata failed')
header = dict(resp.header)
objectSize = int(header.get('content-length'))
partCount = int(objectSize / partSize) if (objectSize % partSize == 0) else int(objectSize / partSize) + 1
if partCount > 10000:
raise Exception('Total parts count should not exceed 10000')
print('Total parts count ' + str(partCount) + '\n')
# Upload multiparts by copy mode
print('Begin to upload multiparts to OBS by copy mode \n')
def handle(self, *args, **options):
# 从OBS查询对象
access_key_id = os.getenv('ACCESS_KEY_ID', '')
secret_access_key = os.getenv('SECRET_ACCESS_KEY', '')
endpoint = os.getenv('OBS_ENDPOINT', '')
bucketName = os.getenv('OBS_BUCKETNAME', '')
if not access_key_id or not secret_access_key or not endpoint or not bucketName:
logger.error('losing required arguments for ObsClient')
sys.exit(1)
obs_client = ObsClient(access_key_id=access_key_id,
secret_access_key=secret_access_key,
server='https://{}'.format(endpoint))
objs = obs_client.listObjects(
bucketName=bucketName)['body']['contents']
# 遍历
if len(objs) == 0:
logger.info('OBS中无对象')
return
for obj in objs:
# 获取对象的地址
object_key = obj['key']
if not object_key.endswith('.mp4'):
continue
# 获取对象的metadata
metadata = obs_client.getObjectMetadata(bucketName, object_key)
metadata_dict = {x: y for x, y in metadata['header']}
# 如果bvid不在metadata_dict中，则下载视频并上传视频至B站
if 'bvid' in metadata_dict:
logger.info('{}已在B站上传，跳过'.format(object_key))
else:
logger.info('{}尚未上传至B站，开始下载'.format(object_key))
# 从OBS下载视频到本地临时目录
videoFile = os.path.join('/tmp', os.path.basename(object_key))
imageFile = videoFile.replace('.mp4', '.png')
if os.path.exists(videoFile):
os.remove(videoFile)
if os.path.exists(imageFile):
os.remove(imageFile)
taskNum = 5
partSize = 10 * 1024 * 1024
enableCheckpoint = True
try:
# 下载视频
resp = obs_client.downloadFile(bucketName, object_key,
videoFile, partSize,
taskNum, enableCheckpoint)
if resp.status < 300:
try:
# 下载封面
img_object_key = object_key.replace('.mp4', '.png')
try:
resp2 = obs_client.downloadFile(
bucketName, img_object_key, imageFile,
partSize, taskNum, enableCheckpoint)
if resp2.status < 300:
# 将下载的视频上传至B站
topic = metadata_dict['meeting_topic']
mid = metadata_dict['meeting_id']
community = metadata_dict['community']
record_start = metadata_dict[
'record_start']
sig = metadata_dict['sig']
date = (datetime.datetime.strptime(
record_start.replace('T', ' ').replace(
'Z', ''), "%Y-%m-%d %H:%M:%S") +
datetime.timedelta(hours=8)
).strftime('%Y-%m-%d')
res = upload(topic, date, videoFile,
imageFile, mid, sig,
community)
try:
if not Record.objects.filter(
mid=mid, platform='bilibili'):
Record.objects.create(
mid=mid, platform='bilibili')
except Exception as e:
logger.error(e)
# 修改metadata
bvid = res['bvid']
agenda = metadata_dict[
'agenda'] if 'agenda' in metadata_dict else ''
record_end = metadata_dict['record_end']
download_url = metadata_dict[
'download_url']
total_size = metadata_dict['total_size']
attenders = metadata_dict['attenders']
metadata = {
"meeting_id": mid,
"meeting_topic": topic,
"community": community,
"sig": sig,
"agenda": agenda,
"record_start": record_start,
"record_end": record_end,
"download_url": download_url,
"total_size": total_size,
"attenders": attenders,
"bvid": bvid
}
try:
resp3 = obs_client.setObjectMetadata(
bucketName, object_key, metadata)
if resp3.status < 300:
logger.info(
'{}: metadata修改成功'.format(
object_key))
else:
logger.error(
'errorCode', resp3.errorCode)
logger.error(
'errorMessage',
resp3.errorMessage)
except:
logger.error(traceback.format_exc())
# 休眠30s避免上传间隔过短
time.sleep(30)
else:
logger.error('errorCode', resp2.errorCode)
logger.error('errorMessage',
resp2.errorMessage)
except Exception as e2:
logger.error(e2)
except:
logger.error(traceback.format_exc())
else:
logger.error('errorCode', resp.errorCode)
logger.error('errorMessage', resp.errorMessage)
except:
logger.error(traceback.format_exc())
# 引入模块
from obs import ObsClient
# 创建ObsClient实例
obsClient = ObsClient(access_key_id='******',
secret_access_key='******',
server='https://obs.cn-north-4.myhuaweicloud.com')
# 使用访问OBS
# resp = obsClient.getBucketMetadata('dli-cn-north-4-aae035c879b649ac93c1b3738fb7d56d')
# print(resp)
resp = obsClient.getObjectMetadata(
'dli-cn-north-4-aae035c879b649ac93c1b3738fb7d56d', 'cat/3.png')
print(resp)
resp = obsClient.deleteObject(
'dli-cn-north-4-aae035c879b649ac93c1b3738fb7d56d', 'cat1/a/')
print(resp)
resp = obsClient.getObjectMetadata(
'dli-cn-north-4-aae035c879b649ac93c1b3738fb7d56d', 'cat1')
print(resp)
# 关闭obsClient
obsClient.close()
