def handle(self, *args, **options):
uid = int(os.getenv('BILI_UID', ''))
if not uid:
logger.error('uid is required')
sys.exit(1)
access_key_id = os.getenv('ACCESS_KEY_ID', '')
secret_access_key = os.getenv('SECRET_ACCESS_KEY', '')
endpoint = os.getenv('OBS_ENDPOINT', '')
bucketName = os.getenv('OBS_BUCKETNAME', '')
if not access_key_id or not secret_access_key or not endpoint or not bucketName:
logger.error('losing required arguments for ObsClient')
sys.exit(1)
# 获取OBS openeuler/meetup/下的MP4列表
obs_client = ObsClient(access_key_id=access_key_id, secret_access_key=secret_access_key,
server='https://{}'.format(endpoint))
objs = obs_client.listObjects(bucketName=bucketName)['body']['contents']
meetup_videos = []
for obj in objs:
if obj['key'].startswith('openeuler/meetup/') and obj['key'].endswith('.mp4'):
meetup_videos.append(obj['key'])
if len(meetup_videos) == 0:
logger.info('no meetup videos in OBS')
return
logger.info('meetup_videos: {}'.format(meetup_videos))
videos = get_videos_g(uid)
bvs = [x['bvid'] for x in videos]
遍历meetup_videos，若obj的metadata无bvid，则下载上传B站
for video in meetup_videos:
metadata = obs_client.getObjectMetadata(bucketName, video)
metadata_dict = {x: y for x, y in metadata['header']}
topic = os.path.basename(video)[:-4]
activity_id = int(video.split('/')[2])
replay_url = 'https://{}.{}/{}'.format(bucketName, endpoint, video)
Activity.objects.filter(id=activity_id).update(replay_url=replay_url)
logger.info('meetup回放视频同步小程序，回放链接: {}'.format(replay_url))
if 'bvid' not in metadata_dict.keys():
# 下载视频
logger.info('开始下载视频')
videoFile = download_video(obs_client, bucketName, video)
logger.info('视频已下载: {}'.format(videoFile))
# 生成封面
logger.info('开始生成封面')
logger.info('topic: {}'.format(topic))
imageFile = generate_cover(topic)
logger.info('封面已生成: {}'.format(imageFile))
# 上传B站
logger.info('开始上传B站')
bvid = upload_to_bilibili(videoFile, imageFile, topic)
logger.info('B站上传成功，bvid: {}'.format(bvid))
# 更新metadata
metadata = {
'bvid': bvid
}
r = obs_client.setObjectMetadata(bucketName, video, metadata)
if r.status < 300:
logger.info('更新metadata')
else:
logger.error(r.errorCode, r.errorMessage)
def doGetObject(lock, completedBlocks, bucketName, objectKey, startPos, endPos,
i):
if IS_WINDOWS:
global obsClient
else:
obsClient = ObsClient(access_key_id=AK,
secret_access_key=SK,
server=server)
resp = obsClient.getObject(bucketName,
objectKey,
headers=GetObjectHeader(range='%d-%d' %
(startPos, endPos)))
if resp.status < 300:
response = resp.body.response
chunk_size = 65536
if response is not None:
with open(localFilePath, 'rb+') as f:
f.seek(startPos, 0)
while True:
chunk = response.read(chunk_size)
if not chunk:
break
f.write(chunk)
response.close()
print('Part#' + str(i + 1) + 'done\n')
with lock:
completedBlocks.value += 1
else:
print('\tPart#' + str(i + 1) + ' failed\n')
def upload_obs(local_path, file_name):
obsClient = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
filemodel = model_path + file_name
bucketName = obs_path.split("/", 1)[0]
resp = obsClient.putFile(bucketName, filemodel, file_path=local_path)
print(resp)
return 0
async def download_data():
AK = 'IJKOCBW6LYDVI6Y1WW0Q'
SK = '7SCcFD6ROXwCAN1B5ios2tapwhvsStZMG1qxrOxy'
server = 'https://obs.ap-southeast-2.myhuaweicloud.com'
bucketName = 'obs-mapper-cie'
objectKey = 'Data/Weather/Bangkok.csv'
localFile = 'Data/Weather/Bangkok.csv'
objectKey2 = 'Data/TrafficCondition/AverageSpeed/AverageSpeed.csv'
localFile2 = 'Data/TrafficCondition/AverageSpeed/AverageSpeed.csv'
objectKey3 = 'Data/Road/RoadInformation.csv'
localFile3 = 'Data/Road/RoadInformation.csv'
obsClient = ObsClient(access_key_id=AK,
secret_access_key=SK,
server=server)
# print('Downloading an object as a socket stream\n')
# print('Downloading an object to :' + localFile + '\n')
resp = obsClient.getObject(bucketName, objectKey, downloadPath=localFile)
resp2 = obsClient.getObject(bucketName,
objectKey2,
downloadPath=localFile2)
resp3 = obsClient.getObject(bucketName,
objectKey3,
downloadPath=localFile3)
# asyncio.run(download_data())
def upload_to_obs(model_local_path):
TestObs = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
bucketName = obs_path.split("/",1)[0]
resultFileName = obs_path.split("/",1)[1] + OBS_MODEL_DIR
TestObs.putFile(bucketName, resultFileName, file_path=model_local_path)
return 0
def __init__(self):
self.key = ""
self.secret = ""
self.obsclient = ObsClient(
access_key_id=self.key,
secret_access_key=self.secret,
server='',
)
def download_obs(local_path, obs_path):
TestObs = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
bucketName = obs_path.split("/", 1)[0]
resultFileName = obs_path.split("/", 1)[1]
resp = TestObs.getObject(bucketName,
resultFileName,
downloadPath=local_path)
if resp.status < 300:
print('Succeeded to download training dataset')
else:
print('Failed to download ')
def doCopyPart(partETags, bucketName, objectKey, partNumber, uploadId, copySource, copySourceRange):
if IS_WINDOWS:
global obsClient
else:
obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server)
resp = obsClient.copyPart(bucketName=bucketName, objectKey=objectKey, partNumber=partNumber, uploadId=uploadId,
copySource=copySource, copySourceRange=copySourceRange)
if resp.status < 300:
partETags[partNumber] = resp.body.etag
print('Part#' + str(partNumber) + 'done\n')
else:
print('\tPart#' + str(partNumber) + ' failed\n')
async def upload_file(objectKey, FilePath):
# Constructs a obs client instance with your account for accessing OBS
obsClient = ObsClient(access_key_id=AK,
secret_access_key=SK,
server=server)
sampleFilePath = 'testUpload.txt'
# Upload an object to your bucket
print('Uploading a new object to OBS from a file\n')
obsClient.putFile(bucketName, objectKey, FilePath)
# asyncio.run(upload_file())
def create_model_dir():
obsClient = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
bucketName = obs_path.split("/", 1)[0]
workDir = obs_path.split("/", 1)[1] + '/model/'
resp_dir = obsClient.putContent(bucketName, workDir, content=None)
if resp_dir.status < 300:
print('Create workDir model success:', resp_dir.requestId)
return workDir
print('Fail to create workDir model and errorCode:', resp_dir.errorCode)
print('errorMessage:', resp_dir.errorMessage)
return ""
def upload_to_obs():
obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
bucket_name = obs_path.split("/", 1)[0]
work_metric = obs_path.split("/", 1)[1] + '/'
model_dir = obs_path.split("/", 1)[1] + '/model/'
model_file = model_dir + MODEL_NAME
config_file = model_dir + CONFIG_NAME
metric_file = work_metric + METRIC_NAME
# upload model to OBS
print_title("upload model to obs !")
obs_client.putFile(bucket_name,
model_file,
file_path=LOCAL_MODEL_PATH + MODEL_NAME)
# upload config file to OBS
print_title("upload config to obs !")
obs_client.putFile(bucket_name, config_file, file_path=LOCAL_CONFIG_PATH)
# upload metric file to OBS
print_title("upload metric to obs !")
obs_client.putFile(bucket_name, metric_file, file_path=LOCAL_METRIC_PATH)
return 0
def get_client(self):
client_type = "OBSClient"
path_style = True if test_config["auth_type"] == "v2" else False
uploadClient = ObsClient(access_key_id=test_config["ak"],
secret_access_key=test_config["sk"],
server=test_config["endpoint"],
is_signature_negotiation=False,
path_style=path_style)
downloadClient = ObsClient(access_key_id=test_config["ak"],
secret_access_key=test_config["sk"],
server=test_config["endpoint"],
is_signature_negotiation=False,
path_style=path_style)
return client_type, uploadClient, downloadClient
def upload_to_obs(tmp_file, activity_id):
access_key_id = os.getenv('ACCESS_KEY_ID', '')
secret_access_key = os.getenv('SECRET_ACCESS_KEY', '')
endpoint = os.getenv('OBS_ENDPOINT', '')
bucketName = os.getenv('OBS_BUCKETNAME_SECOND', '')
if not access_key_id or not secret_access_key or not endpoint or not bucketName:
logger.error('losing required arguments for ObsClient')
sys.exit(1)
obs_client = ObsClient(access_key_id=access_key_id,
secret_access_key=secret_access_key,
server='https://{}'.format(endpoint))
object_key = 'openeuler/miniprogram/activity/{}/sign_url.jpeg'.format(activity_id)
obs_client.uploadFile(bucketName=bucketName, objectKey=object_key, uploadFile=tmp_file, taskNum=10, enableCheckpoint=True)
img_url = 'https://{}.{}/{}'.format(bucketName, endpoint, object_key)
return img_url
class HWOBS:
def __init__(self):
self.obsclient = ObsClient(access_key_id=current_app.config.HW_ACCESS_KEY_ID,
secret_access_key=current_app.config.HW_SECRET_ACCESS_KEY,
server=current_app.config.HW_SERVER)
def mv(self, srcbucketname, src_obj_key, destbucketname, dest_obj_key):
"""移动对象"""
resp = self.obsclient.copyObject(srcbucketname, src_obj_key, destbucketname, dest_obj_key)
if resp.status >= 300:
raise Exception("OBS MV 复制对象出错 errorMessage:", resp.errorMessage, "errorCode:", resp.errorCode)
else:
res = self.obsclient.deleteObject(srcbucketname, src_obj_key)
if res.status >= 300:
raise Exception("OBS MV 删除源对象时出错 errorMessage:", res.errorMessage, "errorCode:", res.errorCode)
def handle(self, *args, **options):
uid = int(os.getenv('BILI_UID', ''))
if not uid:
logger.error('uid is required')
sys.exit(1)
videos = get_videos_g(uid=uid)
# 所有过审视频的bvid集合
bvs = [x['bvid'] for x in videos]
logger.info('所有B站过审视频的bvid: {}'.format(bvs))
logger.info('B站过审视频数: {}'.format(len(bvs)))
access_key_id = os.getenv('ACCESS_KEY_ID', '')
secret_access_key = os.getenv('SECRET_ACCESS_KEY', '')
endpoint = os.getenv('OBS_ENDPOINT', '')
bucketName = os.getenv('OBS_BUCKETNAME', '')
if not access_key_id or not secret_access_key or not endpoint or not bucketName:
logger.error('losing required arguments for ObsClient')
sys.exit(1)
obs_client = ObsClient(access_key_id=access_key_id,
secret_access_key=secret_access_key,
server='https://{}'.format(endpoint))
bili_mids = [
int(x.mid) for x in Record.objects.filter(platform='bilibili',
url__isnull=True)
]
logger.info('所有还未上传B站的会议的mid: {}'.format(bili_mids))
all_bili_mids = [
int(x.mid) for x in Record.objects.filter(platform='bilibili')
]
for mid in all_bili_mids:
obs_record = Record.objects.get(mid=mid, platform='obs')
url = obs_record.url
object_key = url.split('/', 3)[-1]
# 获取对象的metadata
metadata = obs_client.getObjectMetadata(bucketName, object_key)
metadata_dict = {x: y for x, y in metadata['header']}
if 'bvid' not in metadata_dict.keys():
logger.info('meeting {}: 未上传B站，跳过'.format(mid))
else:
logger.info('meeting {}: bvid为{}'.format(
mid, metadata_dict['bvid']))
if metadata_dict['bvid'] not in bvs:
logger.info('meetings: {}: 上传至B站，还未过审'.format(mid))
else:
bili_url = 'https://www.bilibili.com/{}'.format(
metadata_dict['bvid'])
Record.objects.filter(
mid=mid, platform='bilibili').update(url=bili_url)
logger.info('meeting {}: B站已过审，刷新播放地址'.format(mid))
def __init__(self, server, *largs, **kwargs):
""" OBS SFTP server initiate method when auth successfully. includes some step:
1. obtain ak\sk from config;
2. initiate obs client and obs client log;
3. initiate a thread to close idle time greater than idle timeout;
4. if login user has not ak\sk setting, will close resource.
:param server: auth object
:param largs:
:param kwargs:
"""
super(SFTPServerInterface, self).__init__(*largs, **kwargs)
# read ak sk and build client
self._userName = server.username
self._session = server.session
self._transport = server.transport
try:
cfg = config.getConfig()
self._timeout = int(cfg.state.get('timeout'))
self._connHost = '%s_%s' % (server.connIp, server.connPort)
self._endpoint = cfg.auth.get(self._userName).get('obs_endpoint')
ak = cfg.auth.get(self._userName).get('ak')
sk = cfg.auth.get(self._userName).get('sk')
listen_address = cfg.state.get('listen_address')
listen_port = int(cfg.state.get('listen_port'))
self._closeLock = threading.Lock()
self._closed = False
self._ak = ak
self._listenHost = '%s:%s' % (listen_address, listen_port)
self._client = ObsClient(access_key_id=ak, secret_access_key=sk, server=self._endpoint, path_style=True)
self._client.initLog(LogConf(OBS_SDK_LOG_CONFIG), '%s[%s]' % (OBS_SDK_LOG_NAME_PREFIX, self._userName))
self._logClient = logClient
self._logClient.log(logging.INFO, 'Initiate a sftp server for [%s] - ak [%s]', self._userName, ak)
self._ar = {}
self._activeTime = time.time()
self._cleanThread = threading.Thread(target=self._clean, name='trans_%s' % self._connHost).start()
self._obsAdapters = None
except Exception as e:
logClient.log(logging.ERROR, 'Initialize sftp server failed. error message [%s] - %s.', str(e),
traceback.format_exc())
try:
self._transport.close()
if self._cleanThread:
self._cleanThread.close()
except Exception as e:
# ignore
self._logClient.log(logging.WARNING, 'Ignore close error. error message [%s] - %s', str(e),
traceback.format_exc())
def getSplits(self, is_blocklet_split):
from jnius import autoclass
java_list_class = autoclass('java.util.ArrayList')
if str(self.input_split).endswith(".manifest"):
if str(self.input_split).startswith(LOCAL_FILE_PREFIX):
self.manifest_path = str(self.input_split)[len(LOCAL_FILE_PREFIX):]
else:
self.manifest_path = self.input_split
from obs import ObsClient
if str(self.input_split).startswith("s3"):
obsClient = ObsClient(access_key_id=self.ak, secret_access_key=self.sk,
server=str(self.end_point).replace('http://', ''),
long_conn_mode=True)
sources = manifest.getSources(self.manifest_path, CARBON, obsClient)
self.file_path = sources[0]
else:
sources = manifest.getSources(self.manifest_path, CARBON)
java_list = java_list_class()
for source in sources:
java_list.add(source)
return self.ArrowCarbonReaderBuilder.withFileLists(java_list).getSplits(is_blocklet_split)
else:
return self.ArrowCarbonReaderBuilder.getSplits(is_blocklet_split)
def client(self):
if self._client is None:
self._client = ObsClient(
access_key_id=self.info['S3_access_key'],
secret_access_key=self.info['S3_secret_key'],
server=self.info['endpoint'])
return self._client
def download_dataset():
print("Start to download dataset from OBS")
TestObs = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
try:
bucketName = data_path.split("/",1)[0]
resultFileName = data_path.split("/",1)[1] + "/ratings.csv"
resp = TestObs.getObject(bucketName, resultFileName, downloadPath=LOCAL_DATA_DIR)
if resp.status < 300:
print('Succeeded to download training dataset')
else:
print('Failed to download ')
finally:
TestObs.close()
def create_client(ak=None, sk=None, server=None):
is_secure = False if config.client.secure == 'HTTP' else True
return ObsClient(
access_key_id=ak,
secret_access_key=sk,
server=server,
is_secure=is_secure,
)
def download_files_from_obs_concurrently(access_key, secret_key, end_point,
bucket_name, prefix, download_path):
obsClient = ObsClient(access_key_id=access_key,
secret_access_key=secret_key,
server=end_point,
long_conn_mode=True)
files = list_obs_files(obsClient, bucket_name, prefix)
numOfFiles = len(files)
def download(file):
obsClient.getObject(bucket_name, file, download_path + file)
pool = ThreadPool(numOfFiles)
pool.map(download, files)
pool.close()
obsClient.close()
return LOCAL_FILE_PREFIX + download_path + prefix
def download_dataset():
print("Start to download dataset from OBS")
obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
try:
bucket_name = data_path.split("/", 1)[0]
train_file = data_path.split("/", 1)[1] + "/ratings.csv"
resp = obs_client.getObject(bucket_name, train_file, downloadPath=LOCAL_DATA_PATH)
if resp.status < 300:
print('Succeeded to download training dataset')
else:
print('Failed to download ')
raise Exception('Failed to download training dataset from OBS !')
finally:
obs_client.close()
def download_dataset(file_name, local_path):
print("Start to download dataset from OBS")
obs_client = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
try:
bucket_name = data_path.split("/", 1)[0]
train_file = data_path.split("/", 1)[1] + "/" + file_name
response = obs_client.getObject(bucket_name,
train_file,
downloadPath=local_path)
if response.status < 300:
print('succeeded to download file')
else:
print('failed to download file ')
raise Exception('download file from OBS fail.')
finally:
obs_client.close()
def __init__(self, uri, access_key=None, secret_key=None, endpoint=None):
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or access_key
self.secret_key = u.password or secret_key
self.endpoint = endpoint
self.keyname = u.path[1:] # remove first "/"
if access_key is None or secret_key is None or endpoint is None:
raise NotConfigured(
'missing access_key or secret_key or endpoint for huawei cloud'
)
try:
from obs import ObsClient
except ImportError:
raise NotConfigured('missing esdk-obs-python library')
self.obs_client = ObsClient(access_key_id=self.access_key,
secret_access_key=self.secret_key,
server=self.endpoint)
async def upload_file(objectKey, FilePath):
# AK = 'N02JFWTL4SEDYUQSWNLF'
# SK = '9x5ZJ1HoXkI4L9NuOBTdsxQ2eFDh3txBohqGonT8'
# server = 'https://obs.ap-southeast-2.myhuaweicloud.com'
# bucketName = 'obs-mapper'
AK = 'IJKOCBW6LYDVI6Y1WW0Q'
SK = '7SCcFD6ROXwCAN1B5ios2tapwhvsStZMG1qxrOxy'
server = 'https://obs.ap-southeast-2.myhuaweicloud.com'
bucketName = 'obs-mapper-cie'
# Constructs a obs client instance with your account for accessing OBS
obsClient = ObsClient(access_key_id=AK,
secret_access_key=SK,
server=server)
# Upload an object to your bucket
print('Uploading a new object to OBS from a file\n')
obsClient.putFile(bucketName, objectKey, FilePath)
# asyncio.run(upload_file())
def doUploadPart(partETags, bucketName, objectKey, partNumber, uploadId,
filePath, partSize, offset):
if IS_WINDOWS:
global obsClient
else:
obsClient = ObsClient(access_key_id=AK,
secret_access_key=SK,
server=server)
resp = obsClient.uploadPart(bucketName,
objectKey,
partNumber,
uploadId,
content=filePath,
isFile=True,
partSize=partSize,
offset=offset)
if resp.status < 300:
partETags[partNumber] = resp.body.etag
print('Part#' + str(partNumber) + 'done\n')
else:
print('\tPart#' + str(partNumber) + ' failed\n')
def __init__(
self,
access_key_id="9DSDLW5TSCUBU0XHOLG4",
secret_access_key="HB4vH98psY3gvmfZaKDZFs1MyITK8z5jnLKnujkC",
objurl="https://obs.cn-north-4.myhuaweicloud.com",
urlfile="./url.txt",
cdnmap="./cdnMapping.json",
logdir="./log/",
downloaddir="./download",
):
AnsibleApi.__init__(self)
self.key = access_key_id
self.skey = secret_access_key
self.server = objurl
self.urlfile = urlfile
self.cdnmap = cdnmap
self.logdir = logdir
self.downloaddir = downloaddir
self.downloadtask = []
self.obsClient = ObsClient(access_key_id=access_key_id,
secret_access_key=secret_access_key,
server=objurl)
def upload_to_obs():
TestObs = ObsClient(AK, SK, is_secure=True, server=obs_endpoint)
bucketName = obs_path.split("/",1)[0]
workmetric = obs_path.split("/",1)[1] + '/'
workmodel = obs_path.split("/",1)[1] + '/model/'
#workconfig = obs_path.split("/",1)[1] + '/config/'
filemodel = workmodel + MODEL_NAME
fileconfig = workmodel + CONFIG_NAME
filemetric = workmetric + METRIC_NAME
#resultFileName = obs_path.split("/",1)[1] + '/model/xgboost.m'
#configName = obs_path.split("/",1)[1] + '/config/config.json'
TestObs.putContent(bucketName, workmodel, content=None)
#TestObs.putContent(bucketName, workconfig, content=None)
print_title("upload model to obs !")
TestObs.putFile(bucketName, filemodel, file_path=LOCAL_MODEL_DIR)
print_title("upload config to obs !")
TestObs.putFile(bucketName, fileconfig, file_path=LOCAL_CONFIG_DIR)
print_title("upload metric to obs !")
TestObs.putFile(bucketName, filemetric, file_path=LOCAL_METRIC_DIR)
return 0
async def download_file(objectKey,localFile):
# AK = 'N02JFWTL4SEDYUQSWNLF'
# SK = '9x5ZJ1HoXkI4L9NuOBTdsxQ2eFDh3txBohqGonT8'
# server = 'https://obs.ap-southeast-2.myhuaweicloud.com'
# bucketName = 'obs-mapper'
AK = 'IJKOCBW6LYDVI6Y1WW0Q'
SK = '7SCcFD6ROXwCAN1B5ios2tapwhvsStZMG1qxrOxy'
server = 'https://obs.ap-southeast-2.myhuaweicloud.com'
bucketName = 'obs-mapper-cie'
# objectKey = 'Data/Weather/Bangkok.csv'
# localFile = 'Data/Weather/Bangkok.csv'
obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, server=server)
# print('Downloading an object as a socket stream\n')
print('Downloading an object to :' + localFile + '\n')
resp = obsClient.getObject(bucketName, objectKey, downloadPath=localFile)
# asyncio.run(download_file())
def __init__(self,
access_key_id=None,
secret_access_key=None,
server=None,
bucketName=None):
# 创建ObsClient实例
self.base_folder = "/"
if access_key_id == None or secret_access_key == None or server == None:
print(
"# ----- Error invalid OBS server config please check parameter of obs server"
)
exit(0)
else:
self.obsClient = ObsClient(access_key_id=access_key_id,
secret_access_key=secret_access_key,
server=server)
if bucketName != None:
self.bucketName = bucketName
self.bucketClient = self.obsClient.bucketClient(bucketName)
print(
"# ---------------------------------------------------------------------------- #"
)
print(
"# Bucket ToolKit #"
)
print(
"# ---------------------------------------------------------------------------- #"
)
print("# ----access key (AK) : ", access_key_id)
print("# ----secret key (SK): ", secret_access_key)
print("# ----server : ", server)
print("# ----bucket name : ", bucketName)
print("# ----root : ", self.base_folder)
print(
"# ---------------------------------------------------------------------------- #"
)
