def upload_from_buffer(self, region, request, body, size=0):
if not self.ignore_check:
self._prefix_check_and_set_default_val(region, request)
request_str = request.to_json_string()
logger.info("vod upload req = {}, region = {}".format(
request_str, region))
cred = credential.Credential(self.secret_id, self.secret_key)
api_client = vod_client.VodClient(cred, region)
apply_upload_request = models.ApplyUploadRequest()
apply_upload_request.from_json_string(request_str)
apply_upload_response = self.apply_upload(api_client,
apply_upload_request)
logger.info("vod upload ApplyUpload rsp = {}".format(
apply_upload_response.to_json_string()))
if apply_upload_response.TempCertificate is None:
cos_config = CosConfig(
Region=apply_upload_response.StorageRegion,
SecretId=self.secret_id,
SecretKey=self.secret_key)
else:
temp_certificate = apply_upload_response.TempCertificate
cos_config = CosConfig(
Region=apply_upload_response.StorageRegion,
SecretId=temp_certificate.SecretId,
SecretKey=temp_certificate.SecretKey,
Token=temp_certificate.Token)
cos_client = CosS3Client(cos_config)
if StringUtil.is_not_empty(request.MediaType) \
and StringUtil.is_not_empty(apply_upload_response.MediaStoragePath):
self.upload_file_from_buffer(
cos_client, body, apply_upload_response.StorageBucket,
apply_upload_response.MediaStoragePath[1:],
request.ConcurrentUploadNumber)
if StringUtil.is_not_empty(request.CoverType) \
and StringUtil.is_not_empty(apply_upload_response.CoverStoragePath):
self.upload_file_from_buffer(
cos_client, request.CoverFilePath,
apply_upload_response.StorageBucket,
apply_upload_response.CoverStoragePath[1:],
request.ConcurrentUploadNumber)
object_size = self.get_object_size(
cos_client,
apply_upload_response.StorageBucket,
apply_upload_response.MediaStoragePath[1:]
)
if size != 0:
if size != object_size:
logger.error("incomplete upload, src file size: {src_size}, object size: {object_size}".format(
src_size=size, object_size=object_size
))
raise VodClientException("incomplete upload")
commit_upload_request = models.CommitUploadRequest()
commit_upload_request.VodSessionKey = apply_upload_response.VodSessionKey
commit_upload_request.SubAppId = request.SubAppId
commit_upload_response = self.commit_upload(api_client,
commit_upload_request)
commit_upload_response_str = commit_upload_response.to_json_string()
logger.info("vod upload CommitUpload rsp = {}".format(
commit_upload_response_str))
response = VodUploadResponse()
response.from_json_string(commit_upload_response_str)
return response
import sys
import time
import hashlib
import os
import requests
from qcloud_cos import CosS3Client
from qcloud_cos import CosConfig
from qcloud_cos import CosServiceError
SECRET_ID = os.environ["SECRET_ID"]
SECRET_KEY = os.environ["SECRET_KEY"]
test_bucket = "test01-1252448703"
test_object = "test.txt"
special_file_name = "中文" + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
conf = CosConfig(Region="ap-beijing-1",
Secret_id=SECRET_ID,
Secret_key=SECRET_KEY)
client = CosS3Client(conf)
def get_raw_md5(data):
m2 = hashlib.md5(data)
etag = '"' + str(m2.hexdigest()) + '"'
return etag
def gen_file(path, size):
_file = open(path, 'w')
_file.seek(1024 * 1024 * size)
_file.write('cos')
_file.close()
import os
from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
# 配置信息
bucket = os.environ.get("COS_bucket") or 'mnp-1300173558'
secret_id = os.environ.get("TENCENT_APP_ID") or 'xxxx' # 替换为用户的 secretId
secret_key = os.environ.get("TENCENT_APP_KEY") or 'xxxx' # 替换为用户的 secretKey
region = os.environ.get("COS_region") or 'ap-shanghai' # 替换为用户的 Region
token = None # 使用临时密钥需要传入 Token，默认为空，可不填
scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填
config = CosConfig(Region=region,
SecretId=secret_id,
SecretKey=secret_key,
Token=token,
Scheme=scheme)
# 2. 获取客户端对象
client = CosS3Client(config)
import setcos #关于腾讯云对象存储的密钥等数据
from qcloud_cos import CosConfig, CosS3Client
from flask import render_template
from flask import request
import os
import time
import logging
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
log = logging.basicConfig(filename='/var/log/QianDu/QianDuBack.log',
level=logging.DEBUG,
format=LOG_FORMAT)
file_bucket = "file桶name"
sign_bucket = "账户桶name"
config = CosConfig(Region=setcos.region,
SecretId=setcos.secret_id,
SecretKey=setcos.secret_key,
Token=setcos.token,
Scheme=setcos.scheme)
client = CosS3Client(config)
def check(inuser, inpassword, inhash):
if inuser == '':
return '{"code":"400_0","message":"Not User Or Password"}'
if inpassword == '':
return '{"code":"400_0","message":"Not User Or Password"}'
if inhash == '':
return '{"code":"400_1","message":"Hash Error"}'
if len(list(inhash)) < 20:
return '{"code":"400_1","message":"Hash Error"}'
'''hash check'''
# 腾讯云COSV5Python SDK, 目前可以支持Python2.6与Python2.7以及Python3.x
# pip安装指南:pip install -U cos-python-sdk-v5
# cos最新可用地域,参照https://www.qcloud.com/document/product/436/6224
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
# 设置用户属性, 包括secret_id, secret_key, region
# appid已在配置中移除,请在参数Bucket中带上appid。Bucket由bucketname-appid组成
secret_id = 'AKID15IsskiBQACGbAo6WhgcQbVls7HmuG00' # 替换为用户的secret_id
secret_key = 'csivKvxxrMvSvQpMWHuIz12pThQQlWRW' # 替换为用户的secret_key
region = 'ap-beijing-1' # 替换为用户的region
token = '' # 使用临时秘钥需要传入Token，默认为空,可不填
config = CosConfig(Region=region,
Secret_id=secret_id,
Secret_key=secret_key,
Token=token) # 获取配置对象
client = CosS3Client(config)
# 文件流 简单上传
file_name = 'test.txt'
with open('test.txt', 'rb') as fp:
response = client.put_object(
Bucket='test04-123456789', # Bucket由bucketname-appid组成
Body=fp,
Key=file_name,
StorageClass='STANDARD',
ContentType='text/html; charset=utf-8')
print(response['ETag'])
# 字节流 简单上传
ym = time.strftime("%Y/%m", time.localtime())
if oss_type == 'aliyun':
auth = oss2.Auth(secret_id, secret_key)
client = oss2.Bucket(auth, endpoint, bucket)
domain = 'https://' + bucket + '.' + endpoint
for path in args.from_local:
file = path.rsplit("/")[-1]
obj = join(ym, file)
client.put_object_from_file(obj, path)
print(join(domain, obj))
if oss_type == 'tencent':
auth = CosConfig(
Endpoint=endpoint, SecretId=secret_id, SecretKey=secret_key)
client = CosS3Client(auth)
domain = 'https://' + bucket + '.' + endpoint
for path in args.from_local:
file = path.rsplit("/")[-1]
obj = join(ym, file)
client.put_object_from_local_file(
Bucket=bucket,
LocalFilePath=path,
Key=obj,
)
print(join(domain, obj))
def Initialize(main_path: str):
"""
COS 模块初始化，此函数应在所有函数之前调用
:param cfg_path: 配置文件地址。
:param main_path: 程序主目录地址。
"""
# 设置用户属性, 包括secret_id, secret_key, region
# appid已在配置中移除,请在参数Bucket中带上appid。Bucket由bucketname-appid组成
global secret_id, secret_key, region, token, config, client, bucket, cache_time
try:
secret_id = settings.COS_SECRET_ID # 替换为用户的secret_id
secret_key = settings.COS_SECRET_KEY # 替换为用户的secret_key
region = settings.COS_REGION # 替换为用户的region
token = None # 使用临时秘钥需要传入Token，默认为空,可不填
bucket = settings.COS_BUCKET
cache_time = settings.COS_CACHE_TIME # 本地缓存清理时间，以天为计,若参数出错默认为3
print("[COS]secret_id:", secret_id)
print("[COS]secret_key:", secret_key)
print("[COS]region:", region)
print("[COS]bucket:", bucket)
print("[COS]cache_time:{}".format(cache_time))
config = CosConfig(Region=region,
SecretId=secret_id,
SecretKey=secret_key,
Token=token) # 获取配置对象
client = CosS3Client(config)
except Exception as e:
log_cos.error("UnkownError:", e)
print("UnkownError:", e)
log_cos.info("Program Ended")
sys.exit()
global Main_filepath
Main_filepath = main_path
# print(Main_filepath)
global server_mode
server_mode = online_check()
if server_mode:
settings.COS_SERVER_MODE = 1
print("[COS]Online Server is available")
log_cos.info("Cos Server is available")
else:
settings.COS_SERVER_MODE = 0
os.makedirs(os.path.join(settings.MEDIA_ROOT))
os.makedirs(os.path.join(settings.MEDIA_ROOT, "faces_data"),
exist_ok=True)
os.makedirs(os.path.join(settings.MEDIA_ROOT, "tmp"), exist_ok=True)
print("[COS]Online Server is unavailable , using local storage server")
log_cos.warning("Cos is unavailable , using local storage mode")
# todo 接下来将对server_mode进行情况处理
# # 初始化缓存清理 schedule
# global scheduler
# jobstores = {
# 'redis': RedisJobStore(0, host="localhost", port=6379, password=""),
# 'default': SQLAlchemyJobStore(url='sqlite:///data/jobs.sqlite')
# }
# job_defaults = {
# 'coalesce': False,
# 'max_instances': 3
# }
# # BackgroundScheduler: 适合于要求任何在程序后台运行的情况，当希望调度器在应用后台执行时使用。
# scheduler = BackgroundScheduler(jobstores=jobstores, job_defaults=job_defaults, daemonic=False)
# scheduler.remove_all_jobs()
# scheduler.add_job(Auto_DelLocalCache, 'interval', days=cache_time)
# ### weeks,days,hours,minutes,seconds,start_date,end_date,timezone,jitter(最多提前或延迟作业几秒钟)
# scheduler.start()
# print("[COS]Start Background schedule")
log_cos.info("Module COS loaded")
"""
from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
import sys
import json
import logging
import re
from django.conf import settings
'''
the client of cos
'''
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
config = CosConfig(Region=settings.REGION,
SecretId=settings.SECRETID,
SecretKey=settings.SECRETKEY,
Token=settings.TOKEN,
Scheme=settings.SCHEMA)
# 2. 获取客户端对象
client = CosS3Client(config)
'''
function：upload file to Tencent cos
parameters@file:the object of file@file_name:the upload path and file name of file
return@str:the url of file
data:2018/09/06
author:Ricky
'''
def up(file, file_name):
response = client.put_object(Bucket='newsapp-1254170634',
#coding:utf-8
from qcloud_cos import CosConfig
from django.conf import settings
from qcloud_cos import CosS3Client
import sys
import logging
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
region = 'ap-nanjing' # 替换为用户的 Region 区域
config = CosConfig(Region=region,
SecretId=settings.TENCENT_SECRET_ID,
SecretKey=settings.TENCENT_SECRET_KEY)
client = CosS3Client(config)
def create_bucket(name, region):
client.create_bucket(
Bucket="{}-{}".format(name, settings.TENCENT_SECRET_NUMBER))
return "{}-{}".format(name, settings.TENCENT_SECRET_NUMBER)
def upload_file(bucketName, file_object, key):
response = client.upload_file_from_buffer(
Bucket=bucketName,
Body=file_object,
Key=key, #上传之后的名称
)
def get_connection(store_address: dict):
store_address = deepcopy(store_address)
del store_address['storage'], store_address['Bucket']
return CosS3Client(CosConfig(**store_address))
import requests
from qcloud_cos import CosConfig, CosS3Client
from config import cos_config
config = CosConfig(Region=cos_config['Region'],
SecretId=cos_config['SecretId'],
SecretKey=cos_config['SecretKey'])
client = CosS3Client(config)
def all_bucket_detail():
try:
response = client.list_buckets()
return response
except:
return None
def create_bucket(bucket_name):
try:
client.create_bucket(Bucket=f'{bucket_name}-{cos_config["Id"]}')
return True
except:
return False
# {'Key': '1620533083236.jpg',
# 'LastModified': '2021-05-09T10:38:41.000Z',
# 'ETag': '"a65ec2be1c8d3f901c8fb37260fcb9c9"',
def __init__(self):
config = CosConfig(Region=self.region,
SecretId=self.secret_id,
SecretKey=self.secret_key)
self.client = CosS3Client(config)
# 而如果内容发生了变化，使用ETag有助于防止资源的同时更新相互覆盖
import os
import sys
import cv2 as cv
import numpy as np
from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
from qcloud_cos import CosServiceError
from qcloud_cos import CosClientError
from fddeye.tools import logger
from io import BytesIO
from fddeye.config import *
try:
config = CosConfig(Region=COS_REGION, SecretId=COS_SECRET_ID,
SecretKey=COS_SECRET_KEY, Token=COS_TOKEN)
cos_client = CosS3Client(config)
except CosClientError as e:
logger.error(e)
def load_file(file_name,local=False):
if local:
load_local_file(file_name)
else:
with open(file_name, 'rb') as fp:
response = cos_client.put_object(
Bucket=COS_BUCKET,
Body=fp,
Key=file_name,
def __init__(self):
uc = UserConfig()
self._config = CosConfig(Secret_id=uc.secret_id, Secret_key=uc.secret_key,
Region=uc.region, Appid=uc.appid)
self._client = CosS3Client(self._config)
webhook_url = "https://app.qun.qq.com/cgi-bin/api/hookrobot_send"
mirror_url = "https://sm.ms/api/v2/upload"
suo_url = "http://m.6du.in/api_mobileweb_json.php"
# qq群webhook key
Key = os.getenv('QQHook_Key')
# smms key
API_token = os.getenv('smms_Token')
# 腾讯云cos
Secret_id = os.getenv('TX_ID')
Secret_key = os.getenv('TX_Key')
Region = os.getenv('TX_Region')
Bucket = os.getenv('Tx_Bucket')
header = {"Authorization": API_token}
config = CosConfig(Region=Region, SecretId=Secret_id, SecretKey=Secret_key)
client = CosS3Client(config)
def SendtoQQ(name, text):
print("%s:%s" % (name, text))
json_data = {"content": [{"type": 0, "data": ""}]}
json_data["content"][0]["data"] = "%s:\n%s" % (
name, text)
url_params = {'key': Key}
with requests.post(
webhook_url, json=json_data, params=url_params) as respone:
# 不是很清楚原因但是，目前成功发送的http状态码确实是500，200OK反而是失败
if respone.status_code != 500:
print(respone.reason)
def auth(self):
config = CosConfig(Secret_id=self.cloud['SecretId'],
Secret_key=self.cloud['SecretKey'],
Region=self.cloud['Region'])
self.client = CosS3Client(config)
config_y = yaml.load(c)
FILE_RGX = config_y['file_rgx']
DATA_JSON_PATH = config_y['data_json_path']
logging.debug(FILE_RGX)
# 读取上次的 Json 文件
with open(DATA_JSON_PATH, "r", encoding="utf-8") as c:
MODPACK_DATA_OLD = json.load(c)
logging.debug(MODPACK_DATA_OLD)
# 从环境变量中获取相关数据
SECRET_ID = os.environ.get("Secret_id")
SECRET_KEY = os.environ.get("Secret_key")
# 登陆腾讯云
config = CosConfig(Secret_id=SECRET_ID,
Secret_key=SECRET_KEY,
Region='ap-guangzhou')
# 获取客户端对象
client = CosS3Client(config)
#################
# 相关数据的创建 #
#################
# 存储整合数据
MODPACK_DATA_TOTAL = []
# 临时文件夹生成，很遗憾，生成的并不直接是路径
# 而且不允许直接带 .name 获取字符串
TMP_WIN_DIR = tempfile.TemporaryDirectory()
TMP_LINUX_DIR = tempfile.TemporaryDirectory()
def Initialize(cfg_path: str, main_path: str):
"""
COS 模块初始化，此函数应在所有函数之前调用
:param cfg_path: 配置文件地址。
:param main_path: 程序主目录地址。
"""
global cf
cf = configparser.ConfigParser()
cf.read(cfg_path)
# 设置用户属性, 包括secret_id, secret_key, region
# appid已在配置中移除,请在参数Bucket中带上appid。Bucket由bucketname-appid组成
global secret_id, secret_key, region, token, config, client, bucket, cache_time
try:
secret_id = str(cf.get("COS", "secret_id")) # 替换为用户的secret_id
secret_key = str(cf.get("COS", "secret_key")) # 替换为用户的secret_key
region = str(cf.get("COS", "region")) # 替换为用户的region
token = None # 使用临时秘钥需要传入Token，默认为空,可不填
bucket = str(cf.get("COS", "bucket"))
cache_time_str = str(cf.get("COS",
"cache_time")) # 本地缓存清理时间，以天为计,若参数出错默认为3
if cache_time_str.isdigit():
cache_time = int(cache_time_str)
else:
cache_time = 3
print("[COS]secret_id:", secret_id)
print("[COS]secret_key:", secret_key)
print("[COS]region:", region)
print("[COS]bucket:", bucket)
print("[COS]cache_time:{}".format(cache_time))
config = CosConfig(Region=region,
SecretId=secret_id,
SecretKey=secret_key,
Token=token) # 获取配置对象
client = CosS3Client(config)
except Exception as e:
log_cos.error("UnkownError:", e)
print("UnkownError:", e)
log_cos.info("Program Ended")
sys.exit()
global Main_filepath
sysstr = platform.system()
if sysstr == "Windows":
print("[COS]System: Windows")
Main_filepath = main_path
elif sysstr == "Linux":
print("[COS]System: Linux")
Main_filepath = "/tmp"
# print(Main_filepath)
else:
Main_filepath = main_path
global server_mode
server_mode = online_check()
if server_mode:
cf.set("COS", "server_mode", "1")
print("[COS]Online Server is available")
log_cos.info("Cos Server is available")
else:
cf.set("COS", "server_mode", "0")
os.makedirs(os.path.join(Main_filepath, "data", "local", "portrait"),
exist_ok=True)
os.makedirs(os.path.join(Main_filepath, "data", "local", "article"),
exist_ok=True)
print("[COS]Online Server is unavailable , using local storage server")
log_cos.warning("Cos is unavailable , using local storage mode")
# 初始化缓存清理 schedule
global scheduler
if sysstr == "Windows":
jobstores = {
'redis': RedisJobStore(0, host="localhost", port=6379,
password=""),
'default': SQLAlchemyJobStore(url='sqlite:///data/jobs.sqlite')
}
elif sysstr == "Linux":
jobstores = {
'redis': RedisJobStore(0, host="localhost", port=6379,
password=""),
'default': SQLAlchemyJobStore(url='sqlite:///tmp/data/jobs.sqlite')
}
else:
jobstores = {
'redis': RedisJobStore(0, host="localhost", port=6379,
password=""),
'default': SQLAlchemyJobStore(url='sqlite:///data/jobs.sqlite')
}
job_defaults = {'coalesce': False, 'max_instances': 3}
# BackgroundScheduler: 适合于要求任何在程序后台运行的情况，当希望调度器在应用后台执行时使用。
scheduler = BackgroundScheduler(jobstores=jobstores,
job_defaults=job_defaults,
daemonic=False)
scheduler.remove_all_jobs()
scheduler.add_job(Auto_DelLocalCache, 'interval', days=cache_time)
### weeks,days,hours,minutes,seconds,start_date,end_date,timezone,jitter(最多提前或延迟作业几秒钟)
scheduler.start()
print("[COS]Start Background schedule")
log_cos.info("Module COS loaded")
import numpy as np
import cv2
import tempfile
import uuid
import traceback
import yaml
import time
from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
DEBUG = False
cos = yaml.load(open('cos.yaml'))
bucket = cos['bucket']
config = CosConfig(Region=cos['region'], SecretId=cos['secret_id'], SecretKey=cos['secret_key'], Scheme='https')
client = CosS3Client(config)
cache_dir = cos['cache_dir']
def to_image_key(image_file, image_key_or_prefix):
if image_key_or_prefix.endswith('_'):
image_prefix = image_key_or_prefix
image_key = image_prefix + str(uuid.uuid4())
else:
image_key = image_key_or_prefix
if DEBUG:
print('New image key: {}\n{}'.format(image_key, ''.join(traceback.format_stack())))
with open(image_file, 'r') as f:
client.put_object(
Bucket=bucket,
pip install -U cos-python-sdk-v5
$shortcut = (New-Object -COM WScript.Shell).CreateShortcut("$($env:APPDATA)\Microsoft\Windows\SendTo\Tencent COS.lnk")
$shortcut.TargetPath = (Get-Command -Name python | Select-Object -Property Source -First 1).Source
$shortcut.Arguments = (Get-Command -Name upload-to-cos | Select-Object -Property Source -First 1).Source
$shortcut.Save()
[Usage]
Right click a file, then navigate to "send to", then click "COS".
"""
AK = "<Tencent Cloud COS SecretId>"
SK = "<Tencent Cloud COS SecretKey>"
REGION = "ap-shanghai"
BUCKET = "<Bucket Id>-<AppId>" # something like "cdn-123456"
PREFIX = "by-uuid/"
client = CosS3Client(CosConfig(Region=REGION, SecretId=AK, SecretKey=SK))
files = sys.argv[1:]
for file in files:
print("\r\n" + "-" * 40 + "\r\n")
if os.path.isdir(file):
print("Skipping Folder:", file)
continue
print("Uploading File:", file)
extname = file.split(".")[-1]
with open(file, "rb") as f:
target_name = "{}.{}".format(str(uuid.uuid4()), extname)
response = client.put_object(Bucket=BUCKET,
Body=f,
Key=PREFIX + target_name,
StorageClass='STANDARD',
EnableMD5=False)
def upload(self, region, request):
if not self.ignore_check:
self._prefix_check_and_set_default_val(region, request)
request_str = request.to_json_string()
logger.info("vod upload req = {}, region = {}".format(request_str, region))
cred = credential.Credential(self.secret_id, self.secret_key, self.token)
api_client = vod_client.VodClient(cred, region)
parsed_manifest_list = []
segment_file_path_list = []
if self._is_manifest_media_tpe(request.MediaType):
self.parse_manifest(api_client, request.MediaFilePath, request.MediaType, parsed_manifest_list, segment_file_path_list)
apply_upload_request = models.ApplyUploadRequest()
apply_upload_request.from_json_string(request_str)
apply_upload_response = self.apply_upload(api_client, apply_upload_request)
logger.info("vod upload ApplyUpload rsp = {}".format(apply_upload_response.to_json_string()))
if apply_upload_response.TempCertificate is None:
cos_config = CosConfig(
Region=apply_upload_response.StorageRegion,
SecretId=self.secret_id,
SecretKey=self.secret_key
)
else:
temp_certificate = apply_upload_response.TempCertificate
cos_config = CosConfig(
Region=apply_upload_response.StorageRegion,
SecretId=temp_certificate.SecretId,
SecretKey=temp_certificate.SecretKey,
Token=temp_certificate.Token
)
cos_client = CosS3Client(cos_config)
if StringUtil.is_not_empty(request.MediaType) \
and StringUtil.is_not_empty(apply_upload_response.MediaStoragePath):
self.upload_cos(
cos_client,
request.MediaFilePath,
apply_upload_response.StorageBucket,
apply_upload_response.MediaStoragePath[1:],
request.ConcurrentUploadNumber
)
if StringUtil.is_not_empty(request.CoverType) \
and StringUtil.is_not_empty(apply_upload_response.CoverStoragePath):
self.upload_cos(
cos_client,
request.CoverFilePath,
apply_upload_response.StorageBucket,
apply_upload_response.CoverStoragePath[1:],
request.ConcurrentUploadNumber
)
if len(segment_file_path_list) > 0:
for segment_file_path in segment_file_path_list:
storage_dir = os.path.dirname(apply_upload_response.MediaStoragePath)
media_file_dir = os.path.dirname(request.MediaFilePath)
segment_relative_file_path = segment_file_path[len(media_file_dir):].replace("\\", "/")
segment_storage_path = FileUtil.join_path(storage_dir, segment_relative_file_path)
self.upload_cos(
cos_client,
segment_file_path,
apply_upload_response.StorageBucket,
segment_storage_path[1:],
request.ConcurrentUploadNumber
)
commit_upload_request = models.CommitUploadRequest()
commit_upload_request.VodSessionKey = apply_upload_response.VodSessionKey
commit_upload_request.SubAppId = request.SubAppId
commit_upload_response = self.commit_upload(api_client, commit_upload_request)
commit_upload_response_str = commit_upload_response.to_json_string()
logger.info("vod upload CommitUpload rsp = {}".format(commit_upload_response_str))
response = VodUploadResponse()
response.from_json_string(commit_upload_response_str)
return response
import os
from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
from ti.utils import get_temporary_secret_and_token
'''
上传embedding文件到tone平台
'''
#### 指定本地文件路径，可根据需要修改。
local_file = "/home/tione/notebook/embedding/embedding_diff.zip"
#### 用户的存储桶，修改为存放所需数据文件的存储桶，存储桶获取参考腾讯云对象存储
bucket="..."
#### 用户的数据，修改为对应的数据文件路径，文件路径获取参考腾讯云对象存储
data_key="embedding_diff/embedding_diff.zip"
#### 获取用户临时密钥
secret_id, secret_key, token = get_temporary_secret_and_token()
config = CosConfig(Region=os.environ.get('REGION'), SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme='https')
client = CosS3Client(config)
#### 获取文件到本地
response = client.get_object(
Bucket=bucket,
Key=data_key,
)
response['Body'].get_stream_to_file(local_file)
print('finish.....')
from qcloud_image import Client
from qcloud_image import CIUrl, CIFile, CIBuffer, CIUrls, CIFiles, CIBuffers
from PIL import Image
import PIL.Image
import sys
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
appid = 1251762227 # 请替换为您的 APPID
secret_id = u'****' # 请替换为您的 SecretId
secret_key = u'****' # 请替换为您的 SecretKey
region = u'ap-beijing'
token = ''
config = CosConfig(Secret_id=secret_id,
Secret_key=secret_key,
Region=region,
Token=token)
client = CosS3Client(config)
logger = logging.getLogger()
def resize_image(image_path, resized_path):
with Image.open(image_path) as image:
size = float(os.path.getsize(image_path) / 1024 / 1024)
print size
if size >= 1:
image.thumbnail(tuple(x / 2 for x in image.size))
# image.resize((1024,1024),Image.BILINEAR)
image.save(resized_path)
def update(self, *args, **kwargs):
# 支持WEBDAV, MLTALKER
# antvis/experiment_uuid/group/%s_(latest_%d).xxx
if len(args) == 0:
return
local_file_path = os.path.normpath(args[0])
if not os.path.exists(local_file_path):
return
# 扩展名
ext_name = local_file_path.split('.')[-1]
# 文件名
file_name = local_file_path.split('/')[-1]
# 单位M
fsize = os.path.getsize(local_file_path)
fsize = fsize / float(1024 * 1024)
fsize = round(fsize, 2)
# 1.step 获得临时秘钥
response = \
mlogger.getEnv().dashboard.rpc.cos.experiment.get(file_size=fsize,
operator='upload')
if response['status'] == 'ERROR':
logging.error('could get cos credential')
return
tmpSecretId = response['content']['tmpSecretId']
tmpSecretKey = response['content']['tmpSecretKey']
sessionToken = response['content']['sessionToken']
region = response['content']['region']
bucket = response['content']['bucket']
# 2.step 上传文件
tag_str = self.tag if self.group_name == '' else '%s/%s' % (
self.group_name, self.tag)
try:
config = CosConfig(Region=region,
SecretId=tmpSecretId,
SecretKey=tmpSecretKey,
Token=sessionToken,
Scheme='https')
client = CosS3Client(config)
with open(local_file_path, 'rb') as fp:
client.put_object(Bucket=bucket,
Body=fp,
Key='{}/{}/{}/{}'.format(
self.experiment_uuid, tag_str,
self.title, file_name),
StorageClass='STANDARD',
EnableMD5=False)
except:
logging.error('upload {} error'.format(local_file_path))
return
# 3.step 更新平台记录
mlogger.getEnv().dashboard.experiment.patch(
**{
'experiment_name':
self.experiment_name,
'experiment_uuid':
self.experiment_uuid,
'experiment_stage':
mlogger.getEnv().dashboard.experiment_stage,
'experiment_data':
json.dumps({
'FILE_ABSTRACT': {
'group':
tag_str,
'title':
self.title,
'backend':
'ANTVIS',
'path':
'{}/{}/{}/{}'.format(self.experiment_uuid, tag_str,
self.title, file_name),
'size':
fsize,
},
'APP_STAGE':
mlogger.getEnv().dashboard.experiment_stage
})
})
def getPageHtml(self) -> str:
r = requests.post(
'http://10.10.10.2:3001/function',
data=json.dumps({
'context': {
'url': self.url
},
'code':
"module.exports=async({page,context:b})=>{const{url}=b;await page.goto(url,{waitUntil:'networkidle2'});const data=await page.evaluate(()=>{var html=document.getElementsByClassName('notion-page-content')[0].innerHTML;var frag=document.createElement('div');frag.innerHTML=html;var imglist=[].map.call(frag.querySelectorAll('img'),function(img){return img.src});return{html:frag.innerHTML,imglist}});return{data:data,type:'application/json'}};"
}),
headers={'Content-Type': 'application/json'})
if r.status_code != 200:
return "error"
need_del_dict = {
'/image/https': 'https://www.notion.so/image/https',
'href="/': 'href="https://www.notion.so/',
'/images/emoji': 'https://www.notion.so/images/emoji'
}
pattern = re.compile(r'max-width.*?(?:;)')
json_response = r.content.decode()
json_dict = json.loads(json_response)
content = json_dict['html']
need_del_list = pattern.findall(content)
need_del_list = list(dict.fromkeys(need_del_list))
need_del_list.remove('max-width: 100%;')
for item in need_del_list:
need_del_dict[item] = ''
for key in need_del_dict:
content = content.replace(key, need_del_dict[key])
img_pattern = re.compile(
r'https:\/\/www\.notion\.so\/image\/https.*?(?:;cache=v2)')
img_url_list = img_pattern.findall(content)
for img_url in img_url_list:
secret_id = os.getenv('SECRET_ID')
secret_key = os.getenv('SECRET_KEY')
region = os.getenv('REGION')
bucket = os.getenv('BUCKET')
base_url = os.getenv('CDNURL')
token = None
scheme = 'https'
config = CosConfig(Region=region,
SecretId=secret_id,
SecretKey=secret_key,
Token=token,
Scheme=scheme)
client = CosS3Client(config)
r = requests.get(img_url.replace("amp;", ""))
print(img_url.replace("amp;", ""))
print(img_url.replace("amp;", ""))
if r.headers['Content-Type'].find('image') == -1:
continue
file_name = r.headers['Content-Type'] + "/" + str(int(
time.time())) + "." + r.headers['Content-Type'].split('/')[-1]
file_url = f"{base_url}{file_name}"
response = client.put_object(
Bucket=bucket,
Body=r.content,
Key=file_name,
)
print(response)
print(file_url)
content = content.replace(img_url, file_url)
return content
)
print r
def get(client, file_name):
r = client.get_object(
Bucket=bucket,
Key=file_name,
)
r['Body'].get_stream_to_file(file_name)
# r['Body'].get_raw_stream() 流
if __name__ == '__main__':
config = CosConfig(Secret_id=secret_id,
Secret_key=secret_key,
Region=region)
client = CosS3Client(conf=config)
try:
if not os.path.isdir(temp_dir):
os.mkdir(temp_dir)
except IOError, err:
print err
sys.exit()
full_temp = os.path.join(temp_dir, backup_filename)
tar = tarfile.open(name=full_temp, mode='w:gz')
for dirpath, dirnames, filenames in os.walk(backup_dir):
for file in filenames:
full_path = os.path.join(dirpath, file)
from qcloud_cos.crypto import RSAProvider
from qcloud_cos.cos_comm import CiDetectType
SECRET_ID = os.environ["SECRET_ID"]
SECRET_KEY = os.environ["SECRET_KEY"]
TRAVIS_FLAG = os.environ["TRAVIS_FLAG"]
REGION = os.environ["REGION"]
APPID = '1251668577'
test_bucket = 'cos-python-v5-test-' + str(sys.version_info[0]) + '-' + str(
sys.version_info[1]) + '-' + REGION + '-' + APPID
copy_test_bucket = 'copy-' + test_bucket
test_object = "test.txt"
special_file_name = "中文" + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
conf = CosConfig(
Region=REGION,
SecretId=SECRET_ID,
SecretKey=SECRET_KEY,
)
client = CosS3Client(conf, retry=3)
rsa_provider = RSAProvider()
client_for_rsa = CosEncryptionClient(conf, rsa_provider)
aes_provider = AESProvider()
client_for_aes = CosEncryptionClient(conf, aes_provider)
def _create_test_bucket(test_bucket, create_region=None):
try:
if create_region is None:
response = client.create_bucket(Bucket=test_bucket, )
else:
bucket_conf = CosConfig(Region=create_region,
def sync(self):
# 本地文件目录列表
local_file_dir_info = LocalFileDirInfo(self.config.get_local_path())
# 记录的要删除的文件和目录列表
del_file_dict = self.db.file_stat_dict
del_dir_dict = self.db.dir_stat_dict
# 记录的要创建的文件和目录列表
create_file_dict = {}
create_dir_dict = {}
# 文件同步状态校验
for local_file_stat in local_file_dir_info.file_stat_list:
local_file_path = local_file_stat.file_path
# 如果数据库中记录cos中也存在, 则判断下文件是否发生了改变
if (self.db.file_stat_dict.has_key(local_file_path)):
# 如果文件内容未发生变化, 则不用同步, 否则进行同步
if (FileStat.compare(
local_file_stat,
self.db.file_stat_dict[local_file_path]) == 0):
del self.db.file_stat_dict[local_file_path]
else:
create_file_dict[local_file_path] = local_file_stat
# 如果数据库中不存在相关记录, 则插入到要创建的文件列表
else:
create_file_dict[local_file_path] = local_file_stat
# 目录同步状态校验
for local_dir_stat in local_file_dir_info.dir_stat_list:
local_dir_path = local_dir_stat.file_path
# 如果数据库中记录cos中也存在, 则判断下目录是否发生了改变
if (self.db.dir_stat_dict.has_key(local_dir_path)):
del self.db.dir_stat_dict[local_dir_path]
# 如果数据库中不存在相关记录, 则插入到要创建的文件列表
else:
create_dir_dict[local_dir_path] = local_dir_stat
# 生成cos客户端
appid = int(self.config.get_appid())
secret_id = self.config.get_secret_id()
secret_key = self.config.get_secret_key()
bucket = self.config.get_bucket()
timeout = self.config.get_timeout()
cos_client = CosClient(appid, secret_id, secret_key)
cos_config = CosConfig()
cos_config.set_timeout(timeout)
if self.config.get_enable_https() == 1:
cos_config.enable_https()
cos_client.set_config(cos_config)
# 创建要同步的cos 目录
self._create_cos_target_dir(cos_client, bucket)
# 进行同步, 同步顺序如下
# 1 删除文件(之前本地有，现在没有的文件, 或者发生修改的文件)
self._del_cos_sync_file(cos_client, bucket, del_file_dict)
# 2 删除目录(之前本地有，现在没有的目录)
self._del_cos_sync_dir(cos_client, bucket, del_dir_dict)
# 3 创建目录(新建的目录)
self._create_cos_sync_dir(cos_client, bucket, create_dir_dict)
# 4 创建文件(新建的文件)
self._create_cos_sync_file(cos_client, bucket, create_file_dict)
#!/usr/bin/env python
# -*- coding:utf-8 -*-
from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
secret_id = 'XXXXXXXXXXXXXXXX' # 替换为用户的 secretId
secret_key = 'XXXXXXXXXXXXXXXXXXX' # 替换为用户的 secretKey
region = 'ap-hangzhou' # 替换为用户的 Region
config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key)
client = CosS3Client(config)
response = client.create_bucket(
Bucket='test-1251317460',
ACL="public-read" # private / public-read / public-read-write
)
# -*- coding=utf-8
from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
import time
import sys
import logging
from GameNews.settings import COS_SECRETKEY, COS_SECRETID, COS_REGION, COS_BUCKET
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
token = None # 使用临时密钥需要传入Token，默认为空,可不填
config = CosConfig(Region=COS_REGION,
SecretId=COS_SECRETID,
SecretKey=COS_SECRETKEY,
Token=token) # 获取配置对象
client = CosS3Client(config)
def upload_img_byte_stream(file, file_name):
"""
字节流上传文件
"""
ts = time.time()
response = client.put_object(Bucket=COS_BUCKET,
Body=file,
Key='image' + str(ts) + file_name)
return response['ETag']
