def handle(self, *args, **options): uid = int(os.getenv('BILI_UID', '')) if not uid: logger.error('uid is required') sys.exit(1) access_key_id = os.getenv('ACCESS_KEY_ID', '') secret_access_key = os.getenv('SECRET_ACCESS_KEY', '') endpoint = os.getenv('OBS_ENDPOINT', '') bucketName = os.getenv('OBS_BUCKETNAME', '') if not access_key_id or not secret_access_key or not endpoint or not bucketName: logger.error('losing required arguments for ObsClient') sys.exit(1) # 获取OBS openeuler/meetup/下的MP4列表 obs_client = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server='https://{}'.format(endpoint)) objs = obs_client.listObjects(bucketName=bucketName)['body']['contents'] meetup_videos = [] for obj in objs: if obj['key'].startswith('openeuler/meetup/') and obj['key'].endswith('.mp4'): meetup_videos.append(obj['key']) if len(meetup_videos) == 0: logger.info('no meetup videos in OBS') return logger.info('meetup_videos: {}'.format(meetup_videos)) videos = get_videos_g(uid) bvs = [x['bvid'] for x in videos] 遍历meetup_videos,若obj的metadata无bvid,则下载上传B站 for video in meetup_videos: metadata = obs_client.getObjectMetadata(bucketName, video) metadata_dict = {x: y for x, y in metadata['header']} topic = os.path.basename(video)[:-4] activity_id = int(video.split('/')[2]) replay_url = 'https://{}.{}/{}'.format(bucketName, endpoint, video) Activity.objects.filter(id=activity_id).update(replay_url=replay_url) logger.info('meetup回放视频同步小程序,回放链接: {}'.format(replay_url)) if 'bvid' not in metadata_dict.keys(): # 下载视频 logger.info('开始下载视频') videoFile = download_video(obs_client, bucketName, video) logger.info('视频已下载: {}'.format(videoFile)) # 生成封面 logger.info('开始生成封面') logger.info('topic: {}'.format(topic)) imageFile = generate_cover(topic) logger.info('封面已生成: {}'.format(imageFile)) # 上传B站 logger.info('开始上传B站') bvid = upload_to_bilibili(videoFile, imageFile, topic) logger.info('B站上传成功,bvid: {}'.format(bvid)) # 更新metadata metadata = { 'bvid': bvid } r = obs_client.setObjectMetadata(bucketName, video, metadata) if r.status < 300: logger.info('更新metadata') else: logger.error(r.errorCode, r.errorMessage)
keys = [] # First prepare folders and sub folders for i in range(5): key = folderPrefix + str(i) + '/' obsClient.putContent(bucketName, key) keys.append(Object(key)) for j in range(3): subKey = key + subFolderPrefix + str(j) + '/' obsClient.putContent(bucketName, subKey) keys.append(Object(subKey)) # Insert 2 objects in each folder resp = obsClient.listObjects(bucketName) for content in resp.body.contents: for i in range(2): objectKey = content.key + keyPrefix + str(i) obsClient.putContent(bucketName, objectKey, content) keys.append(Object(objectKey)) # Insert 2 objects in root path obsClient.putContent(bucketName, keyPrefix + str(0), content) obsClient.putContent(bucketName, keyPrefix + str(1), content) keys.append(Object(keyPrefix + str(0))) keys.append(Object(keyPrefix + str(1))) print('Put %d objects completed' % len(keys)) print('\n') # List all objects in folder src0/
def run(mid): """ 查询Video根据total_size判断是否需要执行后续操作(下载、上传、保存数据) :param mid: 会议ID :return: """ logger.info('meeting {}: 开始处理'.format(mid)) video = Video.objects.get(mid=mid) # 查询会议的录像信息 recordings = get_recordings(mid) if recordings: total_size = recordings['total_size'] logger.info('meeting {}: 录像文件的总大小为{}'.format(mid, total_size)) # 连接obs服务,实例化ObsClient access_key_id = os.getenv('ACCESS_KEY_ID', '') secret_access_key = os.getenv('SECRET_ACCESS_KEY', '') endpoint = os.getenv('OBS_ENDPOINT', '') bucketName = os.getenv('OBS_BUCKETNAME', '') if not (access_key_id and secret_access_key and endpoint and bucketName): logger.error('losing required arguments for ObsClient') return try: obs_client = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server='https://{}'.format(endpoint)) objs = obs_client.listObjects(bucketName=bucketName) # 预备文件上传路径 start = recordings['recording_files'][0]['recording_start'] month = datetime.datetime.strptime(start.replace('T', ' ').replace('Z', ''), "%Y-%m-%d %H:%M:%S").strftime( "%b").lower() group_name = video.group_name video_name = mid + '.mp4' object_key = 'openeuler/{}/{}/{}/{}'.format(group_name, month, mid, video_name) logger.info('meeting {}: object_key is {}'.format(mid, object_key)) # 收集录像信息待用 end = recordings['recording_files'][0]['recording_end'] zoom_download_url = recordings['recording_files'][0]['download_url'] if not objs['body']['contents']: logger.info('meeting {}: OBS无存储对象,开始下载视频'.format(mid)) topic, filename = download_upload_recordings(start, end, zoom_download_url, mid, total_size, video, endpoint, object_key, group_name, obs_client) return {"mid": mid, "topic": topic, "filename": filename} else: key_size_map = {x['key']: x['size'] for x in objs['body']['contents']} if object_key not in key_size_map.keys(): logger.info('meeting {}: OBS存储服务中无此对象,开始下载视频'.format(mid)) topic, filename = download_upload_recordings(start, end, zoom_download_url, mid, total_size, video, endpoint, object_key, group_name, obs_client) return {"mid": mid, "topic": topic, "filename": filename} elif object_key in key_size_map.keys() and key_size_map[object_key] >= total_size: logger.info('meeting {}: OBS存储服务中已存在该对象且无需替换'.format(mid)) return else: logger.info('meeting {}: OBS存储服务中该对象需要替换,开始下载视频'.format(mid)) topic, filename = download_upload_recordings(start, end, zoom_download_url, mid, total_size, video, endpoint, object_key, group_name, obs_client) return {"mid": mid, "topic": topic, "filename": filename} except Exception as e: logger.error(e)
content = 'Hello OBS' keyPrefix = 'MyObjectKey' keys = [] # First insert 100 objects for demo for i in range(100): key = keyPrefix + str(i) obsClient.putContent(bucketName, key, content) keys.append(Object(key)) print('Put ' + str(len(keys)) + ' objects completed.' + '\n') # List objects using default parameters, will return up to 1000 objects print('List objects using default parameters:\n') resp = obsClient.listObjects(bucketName) for content in resp.body.contents: print('\t' + content.key + ' etag[' + content.etag + ']') print('\n') # List the first 10 objects print('List the first 10 objects :\n') resp = obsClient.listObjects(bucketName, max_keys=10) for content in resp.body.contents: print('\t' + content.key + ' etag[' + content.etag + ']') print('\n') theSecond10ObjectsMarker = resp.body.next_marker
class Objtools(): #初始化配置包括obs key skey def __init__( self, access_key_id="9DSDLW5TSCUBU0XHOLG4", secret_access_key="HB4vH98psY3gvmfZaKDZFs1MyITK8z5jnLKnujkC", objurl="https://obs.cn-north-4.myhuaweicloud.com", urlfile="./url.txt", cdnmap="./cdnMapping.json", logdir="./log/", downloaddir="./download", ): AnsibleApi.__init__(self) self.key = access_key_id self.skey = secret_access_key self.server = objurl self.urlfile = urlfile self.cdnmap = cdnmap self.logdir = logdir self.downloaddir = downloaddir self.downloadtask = [] self.obsClient = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server=objurl) # 文件夹初始化 def initmkdir(self): if os.path.exists(self.downloaddir) == False: print("文件不存在创建目录") os.mkdir(self.downloaddir) if os.path.exists(self.logdir) == False: print("文件不存在创建目录") os.mkdir(self.logdir) #打开文件夹返回句柄 def openfile(self, name, mode): try: f = open(name, mode, encoding='utf-8') except expression as identifier: print(name, mode, "打开文件失败") return f # 写入日志 def logwrite(self, content): f = self.openfile(self.logdir + "/Objtools.log", "a+") localtime = time.asctime(time.localtime(time.time())) f.write("{}------{}------\n".format(localtime, content)) f.close() # http方式下载 def download(self, url, file_path): try: #stram 流方式发开 HTTP1.1后支持 和pipline管道很像 r1 = requests.get(url, stream=True, verify=False) #请求文件得知他的大小判断本地字节 来做一个断点续传 total_size = int(r1.headers['Content-Length']) #请求资源不存在则返回不会下载 if r1.status_code != 200: print("请求错误", url, file_path, r1.status_code, "\n") return except: print("请求失败", url) print("下载失败", file_path) return # 这重要了,先看看本地文件下载了多少 if os.path.exists(file_path): temp_size = os.path.getsize(file_path) # 本地已经下载的文件大小 else: temp_size = 0 # 显示一下下载了多少 print(temp_size) print(total_size) #传给服务器一个范围 服务器会把你只想某一块来拿数据 # 核心部分,这个是请求下载时,从本地文件已经下载过的后面下载 headers = {'Range': 'bytes=%d-' % temp_size} # 重新请求网址,加入新的请求头的 r = requests.get(url, stream=True, verify=False, headers=headers) # "ab"表示追加形式写入文件 with open(file_path, "ab") as f: #文件加锁 保证这个文件只有一个线程在写 其他想写都要等这个线程写完 fcntl.flock(f, fcntl.LOCK_EX) for chunk in r.iter_content(chunk_size=100000): if chunk: temp_size += len(chunk) f.write(chunk) f.flush() ###这是下载实现进度显示#### done = int(50 * temp_size / total_size) sys.stdout.write( "\r[%s%s] %d%%" % ('█' * done, ' ' * (50 - done), 100 * temp_size / total_size)) sys.stdout.flush() #解锁关闭文件 fcntl.flock(f, fcntl.LOCK_UN) f.close() # 获取objs列表 def getobjlist(self, bucketName, objectKey): destfile = {} self.logwrite("获取obj列表") resp = self.obsClient.listObjects(bucketName, objectKey) if resp.status < 300: self.logwrite("请求成功 {} {} ".format(bucketName, objectKey)) try: for i in resp.body.contents: d = str(i).split("/") l = len(d) - 1 if d[l].strip() != "": destfile[d[l]] = True except expression as identifier: self.logwrite("读取obj列表失败") else: self.logwrite('请求失败', bucketName, objectKey) return destfile #下载 拿obs桶中的文件对比哪些该不该下 并且拿url和名字进行一个多协程下载 def DoRun(self, destfile): f = self.openfile(self.urlfile, "rt") #二次去重 _f = set(f.readlines()) for i in _f: l = i.split("http:") n = len(l) - 1 if l[n] == "\n" or l[n] == "": continue url = "http:" + l[n] _n = url.split("/") filename = _n[len(_n) - 1].replace("\n", "").strip() if destfile.get(filename) == None: print("没有就开始下载", url.strip(), filename) self.downloadtask.append( gevent.spawn(self.download, url.strip(), self.downloaddir + "/" + filename.strip())) #开启协程传入一个列表 gevent.joinall(self.downloadtask) # 上传 def upload(self, destfile): f = self.openfile(self.urlfile, "rt") _f = set(f.readlines()) for i in _f: l = i.split("http:") n = len(l) - 1 if l[n] == "\n" or l[n] == "": continue url = "http:" + l[n] _n = url.split("/") filename = _n[len(_n) - 1].replace("\n", "").strip() if destfile.get(filename) == None: try: r1 = requests.get(url, stream=True, verify=False) if r1.status_code == 200: total_size = int(r1.headers['Content-Length']) temp_size = os.path.getsize(self.downloaddir + "/" + filename) # 本地已经下载的文件大小 print(total_size, temp_size) if temp_size >= total_size: print("已下载完成", filename) #去查询obs是否有这个文件 #没有上传 上传后执行更新配置文件 #有不上传 不上传不更新 else: print("未完成下载", filename) except: print("检测文件失败", url, r1.status_code) def upcdnmap(self): #获取上传内容并且更新配置 #ansible同步到各文件 # a = AnsibleApi() # data = a.Arunner("./host.ini", "shell", "echo 1") # ansible_host_path = os.path.join(os.getcwd(), "hosts") # print("=------------", ansible_host_path) # data = a.runner(["192.168.1.207", "192.168.1.208"], # "./host.ini", "shell", "echo 1") # data = a.runner("test", # "./host.ini", "shell", "echo 1") # print(data) # ansible-playbook # a.runner_playbook("./init.yaml", "./host.ini") print("修改cdnmap")
class bucket(): def __init__(self, access_key_id=None, secret_access_key=None, server=None, bucketName=None): # 创建ObsClient实例 self.base_folder = "/" if access_key_id == None or secret_access_key == None or server == None: print( "# ----- Error invalid OBS server config please check parameter of obs server" ) exit(0) else: self.obsClient = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server=server) if bucketName != None: self.bucketName = bucketName self.bucketClient = self.obsClient.bucketClient(bucketName) print( "# ---------------------------------------------------------------------------- #" ) print( "# Bucket ToolKit #" ) print( "# ---------------------------------------------------------------------------- #" ) print("# ----access key (AK) : ", access_key_id) print("# ----secret key (SK): ", secret_access_key) print("# ----server : ", server) print("# ----bucket name : ", bucketName) print("# ----root : ", self.base_folder) print( "# ---------------------------------------------------------------------------- #" ) def getBucketMetadata(self): print('Getting bucket metadata\n') #resp = obsClient.getBucketMetadata(bucketName, origin='http://www.b.com', requestHeaders='Authorization1') resp = self.bucketClient.getBucketMetadata( origin='http://www.b.com', requestHeaders='Authorization1') print('storageClass:', resp.body.storageClass) print('accessContorlAllowOrigin:', resp.body.accessContorlAllowOrigin) print('accessContorlMaxAge:', resp.body.accessContorlMaxAge) print('accessContorlExposeHeaders:', resp.body.accessContorlExposeHeaders) print('accessContorlAllowMethods:', resp.body.accessContorlAllowMethods) print('accessContorlAllowHeaders:', resp.body.accessContorlAllowHeaders) print('Deleting bucket CORS\n') resp = self.bucketClient.deleteBucketCors() print('status' + str(resp.status)) return resp def upload(self, remote_path, local_path): self.obsClient.putFile(self.bucketName, remote_path, local_path) # print("# ===== Uploading ",local_path," ===to : ",remote_path) def download(self, key, download): # print("# ===== Downloading ",key," === to :",download) self.obsClient.getObject(self.bucketName, key, downloadPath=download) def cd(self, folder_key): self.base_folder = folder_key print("# ===== Base Folder", self.base_folder) def delete(self, key): print('# ===== Deleting object ' + key + '\n') self.obsClient.deleteObject(self.bucketName, key) def check(self, key): """ The Sync will overwrite by default. We need check """ assert not self.obsClient.getObject( self.bucketName, key )["status"] < 300, "\n# ===== ERROR : \n# ===== bucket : ({bucketname})\n# ===== key : ({key}) & local upload flow try to overwrite same key".format( bucketname=self.bucketName, key=key) # def mkdir(self,dir): # pass def ls(self, show_item_count=10): print("# ===== list ({path}): ".format(path=self.base_folder)) # resp = self.obsClient.listObjects(self.bucketName) if self.base_folder == "/": resp = self.obsClient.listObjects(self.bucketName) else: resp = self.obsClient.listObjects(self.bucketName, self.base_folder) keylist = [] print("# ===== object count : ", len(resp.body.contents)) for content in resp.body.contents[:show_item_count]: keylist.append(content.key) print(' |--- : ' + content.key + ' etag[' + content.etag + ']') return keylist
def handle(self, *args, **options): # 从OBS查询对象 access_key_id = os.getenv('ACCESS_KEY_ID', '') secret_access_key = os.getenv('SECRET_ACCESS_KEY', '') endpoint = os.getenv('OBS_ENDPOINT', '') bucketName = os.getenv('OBS_BUCKETNAME', '') if not access_key_id or not secret_access_key or not endpoint or not bucketName: logger.error('losing required arguments for ObsClient') sys.exit(1) obs_client = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server='https://{}'.format(endpoint)) objs = obs_client.listObjects( bucketName=bucketName)['body']['contents'] # 遍历 if len(objs) == 0: logger.info('OBS中无对象') return for obj in objs: # 获取对象的地址 object_key = obj['key'] if not object_key.endswith('.mp4'): continue # 获取对象的metadata metadata = obs_client.getObjectMetadata(bucketName, object_key) metadata_dict = {x: y for x, y in metadata['header']} # 如果bvid不在metadata_dict中,则下载视频并上传视频至B站 if 'bvid' in metadata_dict: logger.info('{}已在B站上传,跳过'.format(object_key)) else: logger.info('{}尚未上传至B站,开始下载'.format(object_key)) # 从OBS下载视频到本地临时目录 videoFile = os.path.join('/tmp', os.path.basename(object_key)) imageFile = videoFile.replace('.mp4', '.png') if os.path.exists(videoFile): os.remove(videoFile) if os.path.exists(imageFile): os.remove(imageFile) taskNum = 5 partSize = 10 * 1024 * 1024 enableCheckpoint = True try: # 下载视频 resp = obs_client.downloadFile(bucketName, object_key, videoFile, partSize, taskNum, enableCheckpoint) if resp.status < 300: try: # 下载封面 img_object_key = object_key.replace('.mp4', '.png') try: resp2 = obs_client.downloadFile( bucketName, img_object_key, imageFile, partSize, taskNum, enableCheckpoint) if resp2.status < 300: # 将下载的视频上传至B站 topic = metadata_dict['meeting_topic'] mid = metadata_dict['meeting_id'] community = metadata_dict['community'] record_start = metadata_dict[ 'record_start'] sig = metadata_dict['sig'] date = (datetime.datetime.strptime( record_start.replace('T', ' ').replace( 'Z', ''), "%Y-%m-%d %H:%M:%S") + datetime.timedelta(hours=8) ).strftime('%Y-%m-%d') res = upload(topic, date, videoFile, imageFile, mid, sig, community) try: if not Record.objects.filter( mid=mid, platform='bilibili'): Record.objects.create( mid=mid, platform='bilibili') except Exception as e: logger.error(e) # 修改metadata bvid = res['bvid'] agenda = metadata_dict[ 'agenda'] if 'agenda' in metadata_dict else '' record_end = metadata_dict['record_end'] download_url = metadata_dict[ 'download_url'] total_size = metadata_dict['total_size'] attenders = metadata_dict['attenders'] metadata = { "meeting_id": mid, "meeting_topic": topic, "community": community, "sig": sig, "agenda": agenda, "record_start": record_start, "record_end": record_end, "download_url": download_url, "total_size": total_size, "attenders": attenders, "bvid": bvid } try: resp3 = obs_client.setObjectMetadata( bucketName, object_key, metadata) if resp3.status < 300: logger.info( '{}: metadata修改成功'.format( object_key)) else: logger.error( 'errorCode', resp3.errorCode) logger.error( 'errorMessage', resp3.errorMessage) except: logger.error(traceback.format_exc()) # 休眠30s避免上传间隔过短 time.sleep(30) else: logger.error('errorCode', resp2.errorCode) logger.error('errorMessage', resp2.errorMessage) except Exception as e2: logger.error(e2) except: logger.error(traceback.format_exc()) else: logger.error('errorCode', resp.errorCode) logger.error('errorMessage', resp.errorMessage) except: logger.error(traceback.format_exc())
keys = [] # First prepare folders and sub folders for i in range(5): key = folderPrefix + str(i) + '/' obsClient.putContent(bucketName, key) keys.append(Object(key)) for j in range(3): subKey = key + subFolderPrefix + str(j) + '/' obsClient.putContent(bucketName, subKey) keys.append(Object(subKey)) # Insert 2 objects in each folder resp = obsClient.listObjects(bucketName) for content in resp.body.contents: for i in range(2): objectKey = content.key + keyPrefix + str(i) obsClient.putContent(bucketName, objectKey, content) keys.append(Object(objectKey)) # Insert 2 objects in root path obsClient.putContent(bucketName, keyPrefix + str(0), content) obsClient.putContent(bucketName, keyPrefix + str(1), content) keys.append(Object(keyPrefix + str(0))) keys.append(Object(keyPrefix + str(1))) print('Put %d objects completed' % len(keys)) print('\n') # List versions using default parameters, will return up to 1000 objects
def run(mid): """ 查询Video根据total_size判断是否需要执行后续操作(下载、上传、保存数据) :param mid: 会议ID :return: """ logger.info('meeting {}: 开始处理'.format(mid)) video = Video.objects.get(mid=mid) # 查询会议的录像信息 recordings = get_recordings(mid) if recordings: recordings_list = list( filter(lambda x: x if x['file_extension'] == 'MP4' else None, recordings['recording_files'])) if len(recordings_list) == 0: logger.info('meeting {}: 正在录制中'.format(mid)) return if len(recordings_list) > 1: max_size = max([x['file_size'] for x in recordings_list]) for recording in recordings_list: if recording['file_size'] != max_size: recordings_list.remove(recording) total_size = recordings_list[0]['file_size'] logger.info('meeting {}: 录像文件的总大小为{}'.format(mid, total_size)) # 如果文件过小,则视为无效录像 if total_size < 1024 * 1024 * 10: logger.info('meeting {}: 文件过小,不予操作'.format(mid)) else: # 连接obs服务,实例化ObsClient access_key_id = os.getenv('ACCESS_KEY_ID', '') secret_access_key = os.getenv('SECRET_ACCESS_KEY', '') endpoint = os.getenv('OBS_ENDPOINT', '') bucketName = os.getenv('OBS_BUCKETNAME', '') if not (access_key_id and secret_access_key and endpoint and bucketName): logger.error('losing required arguments for ObsClient') return try: obs_client = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server='https://{}'.format(endpoint)) objs = obs_client.listObjects(bucketName=bucketName) # 预备文件上传路径 start = recordings_list[0]['recording_start'] month = datetime.datetime.strptime( start.replace('T', ' ').replace('Z', ''), "%Y-%m-%d %H:%M:%S").strftime("%b").lower() group_name = video.group_name video_name = mid + '.mp4' object_key = 'openeuler/{}/{}/{}/{}'.format( group_name, month, mid, video_name) logger.info('meeting {}: object_key is {}'.format( mid, object_key)) # 收集录像信息待用 end = recordings_list[0]['recording_end'] zoom_download_url = recordings_list[0]['download_url'] if not objs['body']['contents']: logger.info('meeting {}: OBS无存储对象,开始下载视频'.format(mid)) download_upload_recordings(start, end, zoom_download_url, mid, total_size, video, endpoint, object_key, group_name, obs_client) else: key_size_map = { x['key']: x['size'] for x in objs['body']['contents'] } if object_key not in key_size_map.keys(): logger.info( 'meeting {}: OBS存储服务中无此对象,开始下载视频'.format(mid)) download_upload_recordings(start, end, zoom_download_url, mid, total_size, video, endpoint, object_key, group_name, obs_client) elif object_key in key_size_map.keys( ) and key_size_map[object_key] >= total_size: logger.info( 'meeting {}: OBS存储服务中已存在该对象且无需替换'.format(mid)) else: logger.info( 'meeting {}: OBS存储服务中该对象需要替换,开始下载视频'.format(mid)) download_upload_recordings(start, end, zoom_download_url, mid, total_size, video, endpoint, object_key, group_name, obs_client) except Exception as e: logger.error(e)