def handle(self, *args, **options): uid = int(os.getenv('BILI_UID', '')) if not uid: logger.error('uid is required') sys.exit(1) access_key_id = os.getenv('ACCESS_KEY_ID', '') secret_access_key = os.getenv('SECRET_ACCESS_KEY', '') endpoint = os.getenv('OBS_ENDPOINT', '') bucketName = os.getenv('OBS_BUCKETNAME', '') if not access_key_id or not secret_access_key or not endpoint or not bucketName: logger.error('losing required arguments for ObsClient') sys.exit(1) # 获取OBS openeuler/meetup/下的MP4列表 obs_client = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server='https://{}'.format(endpoint)) objs = obs_client.listObjects(bucketName=bucketName)['body']['contents'] meetup_videos = [] for obj in objs: if obj['key'].startswith('openeuler/meetup/') and obj['key'].endswith('.mp4'): meetup_videos.append(obj['key']) if len(meetup_videos) == 0: logger.info('no meetup videos in OBS') return logger.info('meetup_videos: {}'.format(meetup_videos)) videos = get_videos_g(uid) bvs = [x['bvid'] for x in videos] 遍历meetup_videos,若obj的metadata无bvid,则下载上传B站 for video in meetup_videos: metadata = obs_client.getObjectMetadata(bucketName, video) metadata_dict = {x: y for x, y in metadata['header']} topic = os.path.basename(video)[:-4] activity_id = int(video.split('/')[2]) replay_url = 'https://{}.{}/{}'.format(bucketName, endpoint, video) Activity.objects.filter(id=activity_id).update(replay_url=replay_url) logger.info('meetup回放视频同步小程序,回放链接: {}'.format(replay_url)) if 'bvid' not in metadata_dict.keys(): # 下载视频 logger.info('开始下载视频') videoFile = download_video(obs_client, bucketName, video) logger.info('视频已下载: {}'.format(videoFile)) # 生成封面 logger.info('开始生成封面') logger.info('topic: {}'.format(topic)) imageFile = generate_cover(topic) logger.info('封面已生成: {}'.format(imageFile)) # 上传B站 logger.info('开始上传B站') bvid = upload_to_bilibili(videoFile, imageFile, topic) logger.info('B站上传成功,bvid: {}'.format(bvid)) # 更新metadata metadata = { 'bvid': bvid } r = obs_client.setObjectMetadata(bucketName, video, metadata) if r.status < 300: logger.info('更新metadata') else: logger.error(r.errorCode, r.errorMessage)
def doGetObject(lock, completedBlocks, bucketName, objectKey, startPos, endPos, i): if IS_WINDOWS: global obsClient else: obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) resp = obsClient.getObject(bucketName, objectKey, headers=GetObjectHeader(range='%d-%d' % (startPos, endPos))) if resp.status < 300: response = resp.body.response chunk_size = 65536 if response is not None: with open(localFilePath, 'rb+') as f: f.seek(startPos, 0) while True: chunk = response.read(chunk_size) if not chunk: break f.write(chunk) response.close() print('Part#' + str(i + 1) + 'done\n') with lock: completedBlocks.value += 1 else: print('\tPart#' + str(i + 1) + ' failed\n')
def upload_obs(local_path, file_name): obsClient = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) filemodel = model_path + file_name bucketName = obs_path.split("/", 1)[0] resp = obsClient.putFile(bucketName, filemodel, file_path=local_path) print(resp) return 0
async def download_data(): AK = 'IJKOCBW6LYDVI6Y1WW0Q' SK = '7SCcFD6ROXwCAN1B5ios2tapwhvsStZMG1qxrOxy' server = 'https://obs.ap-southeast-2.myhuaweicloud.com' bucketName = 'obs-mapper-cie' objectKey = 'Data/Weather/Bangkok.csv' localFile = 'Data/Weather/Bangkok.csv' objectKey2 = 'Data/TrafficCondition/AverageSpeed/AverageSpeed.csv' localFile2 = 'Data/TrafficCondition/AverageSpeed/AverageSpeed.csv' objectKey3 = 'Data/Road/RoadInformation.csv' localFile3 = 'Data/Road/RoadInformation.csv' obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) # print('Downloading an object as a socket stream\n') # print('Downloading an object to :' + localFile + '\n') resp = obsClient.getObject(bucketName, objectKey, downloadPath=localFile) resp2 = obsClient.getObject(bucketName, objectKey2, downloadPath=localFile2) resp3 = obsClient.getObject(bucketName, objectKey3, downloadPath=localFile3) # asyncio.run(download_data())
def upload_to_obs(model_local_path): TestObs = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) bucketName = obs_path.split("/",1)[0] resultFileName = obs_path.split("/",1)[1] + OBS_MODEL_DIR TestObs.putFile(bucketName, resultFileName, file_path=model_local_path) return 0
def __init__(self): self.key = "" self.secret = "" self.obsclient = ObsClient( access_key_id=self.key, secret_access_key=self.secret, server='', )
def download_obs(local_path, obs_path): TestObs = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) bucketName = obs_path.split("/", 1)[0] resultFileName = obs_path.split("/", 1)[1] resp = TestObs.getObject(bucketName, resultFileName, downloadPath=local_path) if resp.status < 300: print('Succeeded to download training dataset') else: print('Failed to download ')
def doCopyPart(partETags, bucketName, objectKey, partNumber, uploadId, copySource, copySourceRange): if IS_WINDOWS: global obsClient else: obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) resp = obsClient.copyPart(bucketName=bucketName, objectKey=objectKey, partNumber=partNumber, uploadId=uploadId, copySource=copySource, copySourceRange=copySourceRange) if resp.status < 300: partETags[partNumber] = resp.body.etag print('Part#' + str(partNumber) + 'done\n') else: print('\tPart#' + str(partNumber) + ' failed\n')
async def upload_file(objectKey, FilePath): # Constructs a obs client instance with your account for accessing OBS obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) sampleFilePath = 'testUpload.txt' # Upload an object to your bucket print('Uploading a new object to OBS from a file\n') obsClient.putFile(bucketName, objectKey, FilePath) # asyncio.run(upload_file())
def create_model_dir(): obsClient = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) bucketName = obs_path.split("/", 1)[0] workDir = obs_path.split("/", 1)[1] + '/model/' resp_dir = obsClient.putContent(bucketName, workDir, content=None) if resp_dir.status < 300: print('Create workDir model success:', resp_dir.requestId) return workDir print('Fail to create workDir model and errorCode:', resp_dir.errorCode) print('errorMessage:', resp_dir.errorMessage) return ""
def upload_to_obs(): obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) bucket_name = obs_path.split("/", 1)[0] work_metric = obs_path.split("/", 1)[1] + '/' model_dir = obs_path.split("/", 1)[1] + '/model/' model_file = model_dir + MODEL_NAME config_file = model_dir + CONFIG_NAME metric_file = work_metric + METRIC_NAME # upload model to OBS print_title("upload model to obs !") obs_client.putFile(bucket_name, model_file, file_path=LOCAL_MODEL_PATH + MODEL_NAME) # upload config file to OBS print_title("upload config to obs !") obs_client.putFile(bucket_name, config_file, file_path=LOCAL_CONFIG_PATH) # upload metric file to OBS print_title("upload metric to obs !") obs_client.putFile(bucket_name, metric_file, file_path=LOCAL_METRIC_PATH) return 0
def get_client(self): client_type = "OBSClient" path_style = True if test_config["auth_type"] == "v2" else False uploadClient = ObsClient(access_key_id=test_config["ak"], secret_access_key=test_config["sk"], server=test_config["endpoint"], is_signature_negotiation=False, path_style=path_style) downloadClient = ObsClient(access_key_id=test_config["ak"], secret_access_key=test_config["sk"], server=test_config["endpoint"], is_signature_negotiation=False, path_style=path_style) return client_type, uploadClient, downloadClient
def upload_to_obs(tmp_file, activity_id): access_key_id = os.getenv('ACCESS_KEY_ID', '') secret_access_key = os.getenv('SECRET_ACCESS_KEY', '') endpoint = os.getenv('OBS_ENDPOINT', '') bucketName = os.getenv('OBS_BUCKETNAME_SECOND', '') if not access_key_id or not secret_access_key or not endpoint or not bucketName: logger.error('losing required arguments for ObsClient') sys.exit(1) obs_client = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server='https://{}'.format(endpoint)) object_key = 'openeuler/miniprogram/activity/{}/sign_url.jpeg'.format(activity_id) obs_client.uploadFile(bucketName=bucketName, objectKey=object_key, uploadFile=tmp_file, taskNum=10, enableCheckpoint=True) img_url = 'https://{}.{}/{}'.format(bucketName, endpoint, object_key) return img_url
class HWOBS: def __init__(self): self.obsclient = ObsClient(access_key_id=current_app.config.HW_ACCESS_KEY_ID, secret_access_key=current_app.config.HW_SECRET_ACCESS_KEY, server=current_app.config.HW_SERVER) def mv(self, srcbucketname, src_obj_key, destbucketname, dest_obj_key): """移动对象""" resp = self.obsclient.copyObject(srcbucketname, src_obj_key, destbucketname, dest_obj_key) if resp.status >= 300: raise Exception("OBS MV 复制对象出错 errorMessage:", resp.errorMessage, "errorCode:", resp.errorCode) else: res = self.obsclient.deleteObject(srcbucketname, src_obj_key) if res.status >= 300: raise Exception("OBS MV 删除源对象时出错 errorMessage:", res.errorMessage, "errorCode:", res.errorCode)
def handle(self, *args, **options): uid = int(os.getenv('BILI_UID', '')) if not uid: logger.error('uid is required') sys.exit(1) videos = get_videos_g(uid=uid) # 所有过审视频的bvid集合 bvs = [x['bvid'] for x in videos] logger.info('所有B站过审视频的bvid: {}'.format(bvs)) logger.info('B站过审视频数: {}'.format(len(bvs))) access_key_id = os.getenv('ACCESS_KEY_ID', '') secret_access_key = os.getenv('SECRET_ACCESS_KEY', '') endpoint = os.getenv('OBS_ENDPOINT', '') bucketName = os.getenv('OBS_BUCKETNAME', '') if not access_key_id or not secret_access_key or not endpoint or not bucketName: logger.error('losing required arguments for ObsClient') sys.exit(1) obs_client = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server='https://{}'.format(endpoint)) bili_mids = [ int(x.mid) for x in Record.objects.filter(platform='bilibili', url__isnull=True) ] logger.info('所有还未上传B站的会议的mid: {}'.format(bili_mids)) all_bili_mids = [ int(x.mid) for x in Record.objects.filter(platform='bilibili') ] for mid in all_bili_mids: obs_record = Record.objects.get(mid=mid, platform='obs') url = obs_record.url object_key = url.split('/', 3)[-1] # 获取对象的metadata metadata = obs_client.getObjectMetadata(bucketName, object_key) metadata_dict = {x: y for x, y in metadata['header']} if 'bvid' not in metadata_dict.keys(): logger.info('meeting {}: 未上传B站,跳过'.format(mid)) else: logger.info('meeting {}: bvid为{}'.format( mid, metadata_dict['bvid'])) if metadata_dict['bvid'] not in bvs: logger.info('meetings: {}: 上传至B站,还未过审'.format(mid)) else: bili_url = 'https://www.bilibili.com/{}'.format( metadata_dict['bvid']) Record.objects.filter( mid=mid, platform='bilibili').update(url=bili_url) logger.info('meeting {}: B站已过审,刷新播放地址'.format(mid))
def __init__(self, server, *largs, **kwargs): """ OBS SFTP server initiate method when auth successfully. includes some step: 1. obtain ak\sk from config; 2. initiate obs client and obs client log; 3. initiate a thread to close idle time greater than idle timeout; 4. if login user has not ak\sk setting, will close resource. :param server: auth object :param largs: :param kwargs: """ super(SFTPServerInterface, self).__init__(*largs, **kwargs) # read ak sk and build client self._userName = server.username self._session = server.session self._transport = server.transport try: cfg = config.getConfig() self._timeout = int(cfg.state.get('timeout')) self._connHost = '%s_%s' % (server.connIp, server.connPort) self._endpoint = cfg.auth.get(self._userName).get('obs_endpoint') ak = cfg.auth.get(self._userName).get('ak') sk = cfg.auth.get(self._userName).get('sk') listen_address = cfg.state.get('listen_address') listen_port = int(cfg.state.get('listen_port')) self._closeLock = threading.Lock() self._closed = False self._ak = ak self._listenHost = '%s:%s' % (listen_address, listen_port) self._client = ObsClient(access_key_id=ak, secret_access_key=sk, server=self._endpoint, path_style=True) self._client.initLog(LogConf(OBS_SDK_LOG_CONFIG), '%s[%s]' % (OBS_SDK_LOG_NAME_PREFIX, self._userName)) self._logClient = logClient self._logClient.log(logging.INFO, 'Initiate a sftp server for [%s] - ak [%s]', self._userName, ak) self._ar = {} self._activeTime = time.time() self._cleanThread = threading.Thread(target=self._clean, name='trans_%s' % self._connHost).start() self._obsAdapters = None except Exception as e: logClient.log(logging.ERROR, 'Initialize sftp server failed. error message [%s] - %s.', str(e), traceback.format_exc()) try: self._transport.close() if self._cleanThread: self._cleanThread.close() except Exception as e: # ignore self._logClient.log(logging.WARNING, 'Ignore close error. error message [%s] - %s', str(e), traceback.format_exc())
def getSplits(self, is_blocklet_split): from jnius import autoclass java_list_class = autoclass('java.util.ArrayList') if str(self.input_split).endswith(".manifest"): if str(self.input_split).startswith(LOCAL_FILE_PREFIX): self.manifest_path = str(self.input_split)[len(LOCAL_FILE_PREFIX):] else: self.manifest_path = self.input_split from obs import ObsClient if str(self.input_split).startswith("s3"): obsClient = ObsClient(access_key_id=self.ak, secret_access_key=self.sk, server=str(self.end_point).replace('http://', ''), long_conn_mode=True) sources = manifest.getSources(self.manifest_path, CARBON, obsClient) self.file_path = sources[0] else: sources = manifest.getSources(self.manifest_path, CARBON) java_list = java_list_class() for source in sources: java_list.add(source) return self.ArrowCarbonReaderBuilder.withFileLists(java_list).getSplits(is_blocklet_split) else: return self.ArrowCarbonReaderBuilder.getSplits(is_blocklet_split)
def client(self): if self._client is None: self._client = ObsClient( access_key_id=self.info['S3_access_key'], secret_access_key=self.info['S3_secret_key'], server=self.info['endpoint']) return self._client
def download_dataset(): print("Start to download dataset from OBS") TestObs = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) try: bucketName = data_path.split("/",1)[0] resultFileName = data_path.split("/",1)[1] + "/ratings.csv" resp = TestObs.getObject(bucketName, resultFileName, downloadPath=LOCAL_DATA_DIR) if resp.status < 300: print('Succeeded to download training dataset') else: print('Failed to download ') finally: TestObs.close()
def create_client(ak=None, sk=None, server=None): is_secure = False if config.client.secure == 'HTTP' else True return ObsClient( access_key_id=ak, secret_access_key=sk, server=server, is_secure=is_secure, )
def download_files_from_obs_concurrently(access_key, secret_key, end_point, bucket_name, prefix, download_path): obsClient = ObsClient(access_key_id=access_key, secret_access_key=secret_key, server=end_point, long_conn_mode=True) files = list_obs_files(obsClient, bucket_name, prefix) numOfFiles = len(files) def download(file): obsClient.getObject(bucket_name, file, download_path + file) pool = ThreadPool(numOfFiles) pool.map(download, files) pool.close() obsClient.close() return LOCAL_FILE_PREFIX + download_path + prefix
def download_dataset(): print("Start to download dataset from OBS") obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) try: bucket_name = data_path.split("/", 1)[0] train_file = data_path.split("/", 1)[1] + "/ratings.csv" resp = obs_client.getObject(bucket_name, train_file, downloadPath=LOCAL_DATA_PATH) if resp.status < 300: print('Succeeded to download training dataset') else: print('Failed to download ') raise Exception('Failed to download training dataset from OBS !') finally: obs_client.close()
def download_dataset(file_name, local_path): print("Start to download dataset from OBS") obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) try: bucket_name = data_path.split("/", 1)[0] train_file = data_path.split("/", 1)[1] + "/" + file_name response = obs_client.getObject(bucket_name, train_file, downloadPath=local_path) if response.status < 300: print('succeeded to download file') else: print('failed to download file ') raise Exception('download file from OBS fail.') finally: obs_client.close()
def __init__(self, uri, access_key=None, secret_key=None, endpoint=None): u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key self.secret_key = u.password or secret_key self.endpoint = endpoint self.keyname = u.path[1:] # remove first "/" if access_key is None or secret_key is None or endpoint is None: raise NotConfigured( 'missing access_key or secret_key or endpoint for huawei cloud' ) try: from obs import ObsClient except ImportError: raise NotConfigured('missing esdk-obs-python library') self.obs_client = ObsClient(access_key_id=self.access_key, secret_access_key=self.secret_key, server=self.endpoint)
async def upload_file(objectKey, FilePath): # AK = 'N02JFWTL4SEDYUQSWNLF' # SK = '9x5ZJ1HoXkI4L9NuOBTdsxQ2eFDh3txBohqGonT8' # server = 'https://obs.ap-southeast-2.myhuaweicloud.com' # bucketName = 'obs-mapper' AK = 'IJKOCBW6LYDVI6Y1WW0Q' SK = '7SCcFD6ROXwCAN1B5ios2tapwhvsStZMG1qxrOxy' server = 'https://obs.ap-southeast-2.myhuaweicloud.com' bucketName = 'obs-mapper-cie' # Constructs a obs client instance with your account for accessing OBS obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) # Upload an object to your bucket print('Uploading a new object to OBS from a file\n') obsClient.putFile(bucketName, objectKey, FilePath) # asyncio.run(upload_file())
def doUploadPart(partETags, bucketName, objectKey, partNumber, uploadId, filePath, partSize, offset): if IS_WINDOWS: global obsClient else: obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) resp = obsClient.uploadPart(bucketName, objectKey, partNumber, uploadId, content=filePath, isFile=True, partSize=partSize, offset=offset) if resp.status < 300: partETags[partNumber] = resp.body.etag print('Part#' + str(partNumber) + 'done\n') else: print('\tPart#' + str(partNumber) + ' failed\n')
def __init__( self, access_key_id="9DSDLW5TSCUBU0XHOLG4", secret_access_key="HB4vH98psY3gvmfZaKDZFs1MyITK8z5jnLKnujkC", objurl="https://obs.cn-north-4.myhuaweicloud.com", urlfile="./url.txt", cdnmap="./cdnMapping.json", logdir="./log/", downloaddir="./download", ): AnsibleApi.__init__(self) self.key = access_key_id self.skey = secret_access_key self.server = objurl self.urlfile = urlfile self.cdnmap = cdnmap self.logdir = logdir self.downloaddir = downloaddir self.downloadtask = [] self.obsClient = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server=objurl)
def upload_to_obs(): TestObs = ObsClient(AK, SK, is_secure=True, server=obs_endpoint) bucketName = obs_path.split("/",1)[0] workmetric = obs_path.split("/",1)[1] + '/' workmodel = obs_path.split("/",1)[1] + '/model/' #workconfig = obs_path.split("/",1)[1] + '/config/' filemodel = workmodel + MODEL_NAME fileconfig = workmodel + CONFIG_NAME filemetric = workmetric + METRIC_NAME #resultFileName = obs_path.split("/",1)[1] + '/model/xgboost.m' #configName = obs_path.split("/",1)[1] + '/config/config.json' TestObs.putContent(bucketName, workmodel, content=None) #TestObs.putContent(bucketName, workconfig, content=None) print_title("upload model to obs !") TestObs.putFile(bucketName, filemodel, file_path=LOCAL_MODEL_DIR) print_title("upload config to obs !") TestObs.putFile(bucketName, fileconfig, file_path=LOCAL_CONFIG_DIR) print_title("upload metric to obs !") TestObs.putFile(bucketName, filemetric, file_path=LOCAL_METRIC_DIR) return 0
async def download_file(objectKey,localFile): # AK = 'N02JFWTL4SEDYUQSWNLF' # SK = '9x5ZJ1HoXkI4L9NuOBTdsxQ2eFDh3txBohqGonT8' # server = 'https://obs.ap-southeast-2.myhuaweicloud.com' # bucketName = 'obs-mapper' AK = 'IJKOCBW6LYDVI6Y1WW0Q' SK = '7SCcFD6ROXwCAN1B5ios2tapwhvsStZMG1qxrOxy' server = 'https://obs.ap-southeast-2.myhuaweicloud.com' bucketName = 'obs-mapper-cie' # objectKey = 'Data/Weather/Bangkok.csv' # localFile = 'Data/Weather/Bangkok.csv' obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server) # print('Downloading an object as a socket stream\n') print('Downloading an object to :' + localFile + '\n') resp = obsClient.getObject(bucketName, objectKey, downloadPath=localFile) # asyncio.run(download_file())
def __init__(self, access_key_id=None, secret_access_key=None, server=None, bucketName=None): # 创建ObsClient实例 self.base_folder = "/" if access_key_id == None or secret_access_key == None or server == None: print( "# ----- Error invalid OBS server config please check parameter of obs server" ) exit(0) else: self.obsClient = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key, server=server) if bucketName != None: self.bucketName = bucketName self.bucketClient = self.obsClient.bucketClient(bucketName) print( "# ---------------------------------------------------------------------------- #" ) print( "# Bucket ToolKit #" ) print( "# ---------------------------------------------------------------------------- #" ) print("# ----access key (AK) : ", access_key_id) print("# ----secret key (SK): ", secret_access_key) print("# ----server : ", server) print("# ----bucket name : ", bucketName) print("# ----root : ", self.base_folder) print( "# ---------------------------------------------------------------------------- #" )