def handle(self, *args, **options): uid = int(os.getenv('BILI_UID', '')) if not uid: logger.error('uid is required') sys.exit(1) access_key_id = os.getenv('ACCESS_KEY_ID', '') secret_access_key = os.getenv('SECRET_ACCESS_KEY', '') endpoint = os.getenv('OBS_ENDPOINT', '') bucketName = os.getenv('OBS_BUCKETNAME', '') if not access_key_id or not secret_access_key or not endpoint or not bucketName: logger.error('losing required arguments for ObsClient') sys.exit(1) # 获取OBS openeuler/meetup/下的MP4列表 obs_client = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server='https://{}'.format(endpoint)) objs = obs_client.listObjects(bucketName=bucketName)['body']['contents'] meetup_videos = [] for obj in objs: if obj['key'].startswith('openeuler/meetup/') and obj['key'].endswith('.mp4'): meetup_videos.append(obj['key']) if len(meetup_videos) == 0: logger.info('no meetup videos in OBS') return logger.info('meetup_videos: {}'.format(meetup_videos)) videos = get_videos_g(uid) bvs = [x['bvid'] for x in videos] 遍历meetup_videos,若obj的metadata无bvid,则下载上传B站 for video in meetup_videos: metadata = obs_client.getObjectMetadata(bucketName, video) metadata_dict = {x: y for x, y in metadata['header']} topic = os.path.basename(video)[:-4] activity_id = int(video.split('/')[2]) replay_url = 'https://{}.{}/{}'.format(bucketName, endpoint, video) Activity.objects.filter(id=activity_id).update(replay_url=replay_url) logger.info('meetup回放视频同步小程序,回放链接: {}'.format(replay_url)) if 'bvid' not in metadata_dict.keys(): # 下载视频 logger.info('开始下载视频') videoFile = download_video(obs_client, bucketName, video) logger.info('视频已下载: {}'.format(videoFile)) # 生成封面 logger.info('开始生成封面') logger.info('topic: {}'.format(topic)) imageFile = generate_cover(topic) logger.info('封面已生成: {}'.format(imageFile)) # 上传B站 logger.info('开始上传B站') bvid = upload_to_bilibili(videoFile, imageFile, topic) logger.info('B站上传成功,bvid: {}'.format(bvid)) # 更新metadata metadata = { 'bvid': bvid } r = obs_client.setObjectMetadata(bucketName, video, metadata) if r.status < 300: logger.info('更新metadata') else: logger.error(r.errorCode, r.errorMessage)
def handle(self, *args, **options): uid = int(os.getenv('BILI_UID', '')) if not uid: logger.error('uid is required') sys.exit(1) videos = get_videos_g(uid=uid) # 所有过审视频的bvid集合 bvs = [x['bvid'] for x in videos] logger.info('所有B站过审视频的bvid: {}'.format(bvs)) logger.info('B站过审视频数: {}'.format(len(bvs))) access_key_id = os.getenv('ACCESS_KEY_ID', '') secret_access_key = os.getenv('SECRET_ACCESS_KEY', '') endpoint = os.getenv('OBS_ENDPOINT', '') bucketName = os.getenv('OBS_BUCKETNAME', '') if not access_key_id or not secret_access_key or not endpoint or not bucketName: logger.error('losing required arguments for ObsClient') sys.exit(1) obs_client = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server='https://{}'.format(endpoint)) bili_mids = [ int(x.mid) for x in Record.objects.filter(platform='bilibili', url__isnull=True) ] logger.info('所有还未上传B站的会议的mid: {}'.format(bili_mids)) all_bili_mids = [ int(x.mid) for x in Record.objects.filter(platform='bilibili') ] for mid in all_bili_mids: obs_record = Record.objects.get(mid=mid, platform='obs') url = obs_record.url object_key = url.split('/', 3)[-1] # 获取对象的metadata metadata = obs_client.getObjectMetadata(bucketName, object_key) metadata_dict = {x: y for x, y in metadata['header']} if 'bvid' not in metadata_dict.keys(): logger.info('meeting {}: 未上传B站,跳过'.format(mid)) else: logger.info('meeting {}: bvid为{}'.format( mid, metadata_dict['bvid'])) if metadata_dict['bvid'] not in bvs: logger.info('meetings: {}: 上传至B站,还未过审'.format(mid)) else: bili_url = 'https://www.bilibili.com/{}'.format( metadata_dict['bvid']) Record.objects.filter( mid=mid, platform='bilibili').update(url=bili_url) logger.info('meeting {}: B站已过审,刷新播放地址'.format(mid))
# Create bucket print('Create a new bucket for demo\n') obsClient.createBucket(bucketName) # Setting object mime type headers = PutObjectHeader(contentType='text/plain') # Setting self-defined metadata metadata = {'meta1': 'value1', 'meta2': 'value2'} resp = obsClient.putContent(bucketName, objectKey, 'Hello OBS', metadata=metadata, headers=headers) if resp.status < 300: print('Create object ' + objectKey + ' successfully!\n') else: print('common msg:status:', resp.status, ',errorCode:', resp.errorCode, ',errorMessage:', resp.errorMessage) # Get object metadata resp = obsClient.getObjectMetadata(bucketName, objectKey) header = dict(resp.header) print('\tContentType:' + header.get('content-type')) print('\tmeta1:' + header.get('meta1')) print('\tmeta2:' + header.get('meta2')) obsClient.deleteObject(bucketName, objectKey)
# get ak/sk credentials credentials = create_temporary_ak_sk(token) AK = credentials["access"] SK = credentials["secret"] # Constructs a obs client instance with your account for accessing OBS obs_client = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) bucket_client = obs_client.bucketClient(bucket_name) # TODO create OBS folders (not working) base_folder = f"{uid}/" obs_client.putContent(bucket_name, base_folder) resp = obs_client.getObjectMetadata(bucket_name, base_folder) print('Size of the empty folder ' + base_folder + ' is ' + str(dict(resp.header).get('content-length'))) code_folder = f"{base_folder}code/" obs_client.putContent(bucket_name, code_folder) data_folder = f"{base_folder}data/" obs_client.putContent(bucket_name, data_folder) model_folder = f"{base_folder}model/" obs_client.putContent(bucket_name, model_folder) # upload script and data to OBS train_file = code_folder + "train.py" obs_client.putFile(bucket_name, train_file, "03-detect-annotation-anomalies.py")
from __future__ import print_function from obs import ObsClient AK = '*** Provide your Access Key ***' SK = '*** Provide your Secret Key ***' server = 'https://your-endpoint' bucketName = 'my-obs-bucket-demo' # Constructs a obs client instance with your account for accessing OBS obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) # Create bucket print('Create a new bucket for demo\n') obsClient.createBucket(bucketName) keySuffixWithSlash1 = 'MyObjectKey1/' keySuffixWithSlash2 = 'MyObjectKey2/' # Create two empty folder without request body, note that the key must be suffixed with a slash obsClient.putContent(bucketName, keySuffixWithSlash1, '') print('Creating an empty folder ' + keySuffixWithSlash1) obsClient.putContent(bucketName, keySuffixWithSlash2) print('Creating an empty folder ' + keySuffixWithSlash2) # Verify whether the size of the empty folder is zero resp = obsClient.getObjectMetadata(bucketName, keySuffixWithSlash1) print('Size of the empty folder ' + keySuffixWithSlash1 + ' is ' + str(dict(resp.header).get('content-length'))) resp = obsClient.getObjectMetadata(bucketName, keySuffixWithSlash2) print('Size of the empty folder ' + keySuffixWithSlash2 + ' is ' + str(dict(resp.header).get('content-length')))
print('Uploading a new object to OBS from a file\n') resp = obsClient.putFile(sourceBucketName, sourceObjectKey, sampleFilePath) if resp.status >= 300: raise Exception('putFile failed') # Claim a upload id firstly resp = obsClient.initiateMultipartUpload(bucketName, objectKey) if resp.status >= 300: raise Exception('initiateMultipartUpload failed') uploadId = resp.body.uploadId print('Claiming a new upload id ' + uploadId + '\n') # 5MB partSize = 5 * 1024 * 1024 resp = obsClient.getObjectMetadata(sourceBucketName, sourceObjectKey) if resp.status >= 300: raise Exception('getObjectMetadata failed') header = dict(resp.header) objectSize = int(header.get('content-length')) partCount = int(objectSize / partSize) if (objectSize % partSize == 0) else int(objectSize / partSize) + 1 if partCount > 10000: raise Exception('Total parts count should not exceed 10000') print('Total parts count ' + str(partCount) + '\n') # Upload multiparts by copy mode print('Begin to upload multiparts to OBS by copy mode \n')
def handle(self, *args, **options): # 从OBS查询对象 access_key_id = os.getenv('ACCESS_KEY_ID', '') secret_access_key = os.getenv('SECRET_ACCESS_KEY', '') endpoint = os.getenv('OBS_ENDPOINT', '') bucketName = os.getenv('OBS_BUCKETNAME', '') if not access_key_id or not secret_access_key or not endpoint or not bucketName: logger.error('losing required arguments for ObsClient') sys.exit(1) obs_client = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server='https://{}'.format(endpoint)) objs = obs_client.listObjects( bucketName=bucketName)['body']['contents'] # 遍历 if len(objs) == 0: logger.info('OBS中无对象') return for obj in objs: # 获取对象的地址 object_key = obj['key'] if not object_key.endswith('.mp4'): continue # 获取对象的metadata metadata = obs_client.getObjectMetadata(bucketName, object_key) metadata_dict = {x: y for x, y in metadata['header']} # 如果bvid不在metadata_dict中,则下载视频并上传视频至B站 if 'bvid' in metadata_dict: logger.info('{}已在B站上传,跳过'.format(object_key)) else: logger.info('{}尚未上传至B站,开始下载'.format(object_key)) # 从OBS下载视频到本地临时目录 videoFile = os.path.join('/tmp', os.path.basename(object_key)) imageFile = videoFile.replace('.mp4', '.png') if os.path.exists(videoFile): os.remove(videoFile) if os.path.exists(imageFile): os.remove(imageFile) taskNum = 5 partSize = 10 * 1024 * 1024 enableCheckpoint = True try: # 下载视频 resp = obs_client.downloadFile(bucketName, object_key, videoFile, partSize, taskNum, enableCheckpoint) if resp.status < 300: try: # 下载封面 img_object_key = object_key.replace('.mp4', '.png') try: resp2 = obs_client.downloadFile( bucketName, img_object_key, imageFile, partSize, taskNum, enableCheckpoint) if resp2.status < 300: # 将下载的视频上传至B站 topic = metadata_dict['meeting_topic'] mid = metadata_dict['meeting_id'] community = metadata_dict['community'] record_start = metadata_dict[ 'record_start'] sig = metadata_dict['sig'] date = (datetime.datetime.strptime( record_start.replace('T', ' ').replace( 'Z', ''), "%Y-%m-%d %H:%M:%S") + datetime.timedelta(hours=8) ).strftime('%Y-%m-%d') res = upload(topic, date, videoFile, imageFile, mid, sig, community) try: if not Record.objects.filter( mid=mid, platform='bilibili'): Record.objects.create( mid=mid, platform='bilibili') except Exception as e: logger.error(e) # 修改metadata bvid = res['bvid'] agenda = metadata_dict[ 'agenda'] if 'agenda' in metadata_dict else '' record_end = metadata_dict['record_end'] download_url = metadata_dict[ 'download_url'] total_size = metadata_dict['total_size'] attenders = metadata_dict['attenders'] metadata = { "meeting_id": mid, "meeting_topic": topic, "community": community, "sig": sig, "agenda": agenda, "record_start": record_start, "record_end": record_end, "download_url": download_url, "total_size": total_size, "attenders": attenders, "bvid": bvid } try: resp3 = obs_client.setObjectMetadata( bucketName, object_key, metadata) if resp3.status < 300: logger.info( '{}: metadata修改成功'.format( object_key)) else: logger.error( 'errorCode', resp3.errorCode) logger.error( 'errorMessage', resp3.errorMessage) except: logger.error(traceback.format_exc()) # 休眠30s避免上传间隔过短 time.sleep(30) else: logger.error('errorCode', resp2.errorCode) logger.error('errorMessage', resp2.errorMessage) except Exception as e2: logger.error(e2) except: logger.error(traceback.format_exc()) else: logger.error('errorCode', resp.errorCode) logger.error('errorMessage', resp.errorMessage) except: logger.error(traceback.format_exc())
# 引入模块 from obs import ObsClient # 创建ObsClient实例 obsClient = ObsClient(access_key_id='******', secret_access_key='******', server='https://obs.cn-north-4.myhuaweicloud.com') # 使用访问OBS # resp = obsClient.getBucketMetadata('dli-cn-north-4-aae035c879b649ac93c1b3738fb7d56d') # print(resp) resp = obsClient.getObjectMetadata( 'dli-cn-north-4-aae035c879b649ac93c1b3738fb7d56d', 'cat/3.png') print(resp) resp = obsClient.deleteObject( 'dli-cn-north-4-aae035c879b649ac93c1b3738fb7d56d', 'cat1/a/') print(resp) resp = obsClient.getObjectMetadata( 'dli-cn-north-4-aae035c879b649ac93c1b3738fb7d56d', 'cat1') print(resp) # 关闭obsClient obsClient.close()