def upload_from_buffer(self, region, request, body, size=0): if not self.ignore_check: self._prefix_check_and_set_default_val(region, request) request_str = request.to_json_string() logger.info("vod upload req = {}, region = {}".format( request_str, region)) cred = credential.Credential(self.secret_id, self.secret_key) api_client = vod_client.VodClient(cred, region) apply_upload_request = models.ApplyUploadRequest() apply_upload_request.from_json_string(request_str) apply_upload_response = self.apply_upload(api_client, apply_upload_request) logger.info("vod upload ApplyUpload rsp = {}".format( apply_upload_response.to_json_string())) if apply_upload_response.TempCertificate is None: cos_config = CosConfig( Region=apply_upload_response.StorageRegion, SecretId=self.secret_id, SecretKey=self.secret_key) else: temp_certificate = apply_upload_response.TempCertificate cos_config = CosConfig( Region=apply_upload_response.StorageRegion, SecretId=temp_certificate.SecretId, SecretKey=temp_certificate.SecretKey, Token=temp_certificate.Token) cos_client = CosS3Client(cos_config) if StringUtil.is_not_empty(request.MediaType) \ and StringUtil.is_not_empty(apply_upload_response.MediaStoragePath): self.upload_file_from_buffer( cos_client, body, apply_upload_response.StorageBucket, apply_upload_response.MediaStoragePath[1:], request.ConcurrentUploadNumber) if StringUtil.is_not_empty(request.CoverType) \ and StringUtil.is_not_empty(apply_upload_response.CoverStoragePath): self.upload_file_from_buffer( cos_client, request.CoverFilePath, apply_upload_response.StorageBucket, apply_upload_response.CoverStoragePath[1:], request.ConcurrentUploadNumber) object_size = self.get_object_size( cos_client, apply_upload_response.StorageBucket, apply_upload_response.MediaStoragePath[1:] ) if size != 0: if size != object_size: logger.error("incomplete upload, src file size: {src_size}, object size: {object_size}".format( src_size=size, object_size=object_size )) raise VodClientException("incomplete upload") commit_upload_request = models.CommitUploadRequest() commit_upload_request.VodSessionKey = apply_upload_response.VodSessionKey commit_upload_request.SubAppId = request.SubAppId commit_upload_response = self.commit_upload(api_client, commit_upload_request) commit_upload_response_str = commit_upload_response.to_json_string() logger.info("vod upload CommitUpload rsp = {}".format( commit_upload_response_str)) response = VodUploadResponse() response.from_json_string(commit_upload_response_str) return response
import sys import time import hashlib import os import requests from qcloud_cos import CosS3Client from qcloud_cos import CosConfig from qcloud_cos import CosServiceError SECRET_ID = os.environ["SECRET_ID"] SECRET_KEY = os.environ["SECRET_KEY"] test_bucket = "test01-1252448703" test_object = "test.txt" special_file_name = "中文" + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" conf = CosConfig(Region="ap-beijing-1", Secret_id=SECRET_ID, Secret_key=SECRET_KEY) client = CosS3Client(conf) def get_raw_md5(data): m2 = hashlib.md5(data) etag = '"' + str(m2.hexdigest()) + '"' return etag def gen_file(path, size): _file = open(path, 'w') _file.seek(1024 * 1024 * size) _file.write('cos') _file.close()
import os from qcloud_cos import CosConfig from qcloud_cos import CosS3Client # 配置信息 bucket = os.environ.get("COS_bucket") or 'mnp-1300173558' secret_id = os.environ.get("TENCENT_APP_ID") or 'xxxx' # 替换为用户的 secretId secret_key = os.environ.get("TENCENT_APP_KEY") or 'xxxx' # 替换为用户的 secretKey region = os.environ.get("COS_region") or 'ap-shanghai' # 替换为用户的 Region token = None # 使用临时密钥需要传入 Token,默认为空,可不填 scheme = 'https' # 指定使用 http/https 协议来访问 COS,默认为 https,可不填 config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) # 2. 获取客户端对象 client = CosS3Client(config)
import setcos #关于腾讯云对象存储的密钥等数据 from qcloud_cos import CosConfig, CosS3Client from flask import render_template from flask import request import os import time import logging LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s" log = logging.basicConfig(filename='/var/log/QianDu/QianDuBack.log', level=logging.DEBUG, format=LOG_FORMAT) file_bucket = "file桶name" sign_bucket = "账户桶name" config = CosConfig(Region=setcos.region, SecretId=setcos.secret_id, SecretKey=setcos.secret_key, Token=setcos.token, Scheme=setcos.scheme) client = CosS3Client(config) def check(inuser, inpassword, inhash): if inuser == '': return '{"code":"400_0","message":"Not User Or Password"}' if inpassword == '': return '{"code":"400_0","message":"Not User Or Password"}' if inhash == '': return '{"code":"400_1","message":"Hash Error"}' if len(list(inhash)) < 20: return '{"code":"400_1","message":"Hash Error"}' '''hash check'''
# 腾讯云COSV5Python SDK, 目前可以支持Python2.6与Python2.7以及Python3.x # pip安装指南:pip install -U cos-python-sdk-v5 # cos最新可用地域,参照https://www.qcloud.com/document/product/436/6224 logging.basicConfig(level=logging.INFO, stream=sys.stdout) # 设置用户属性, 包括secret_id, secret_key, region # appid已在配置中移除,请在参数Bucket中带上appid。Bucket由bucketname-appid组成 secret_id = 'AKID15IsskiBQACGbAo6WhgcQbVls7HmuG00' # 替换为用户的secret_id secret_key = 'csivKvxxrMvSvQpMWHuIz12pThQQlWRW' # 替换为用户的secret_key region = 'ap-beijing-1' # 替换为用户的region token = '' # 使用临时秘钥需要传入Token,默认为空,可不填 config = CosConfig(Region=region, Secret_id=secret_id, Secret_key=secret_key, Token=token) # 获取配置对象 client = CosS3Client(config) # 文件流 简单上传 file_name = 'test.txt' with open('test.txt', 'rb') as fp: response = client.put_object( Bucket='test04-123456789', # Bucket由bucketname-appid组成 Body=fp, Key=file_name, StorageClass='STANDARD', ContentType='text/html; charset=utf-8') print(response['ETag']) # 字节流 简单上传
ym = time.strftime("%Y/%m", time.localtime()) if oss_type == 'aliyun': auth = oss2.Auth(secret_id, secret_key) client = oss2.Bucket(auth, endpoint, bucket) domain = 'https://' + bucket + '.' + endpoint for path in args.from_local: file = path.rsplit("/")[-1] obj = join(ym, file) client.put_object_from_file(obj, path) print(join(domain, obj)) if oss_type == 'tencent': auth = CosConfig( Endpoint=endpoint, SecretId=secret_id, SecretKey=secret_key) client = CosS3Client(auth) domain = 'https://' + bucket + '.' + endpoint for path in args.from_local: file = path.rsplit("/")[-1] obj = join(ym, file) client.put_object_from_local_file( Bucket=bucket, LocalFilePath=path, Key=obj, ) print(join(domain, obj))
def Initialize(main_path: str): """ COS 模块初始化,此函数应在所有函数之前调用 :param cfg_path: 配置文件地址。 :param main_path: 程序主目录地址。 """ # 设置用户属性, 包括secret_id, secret_key, region # appid已在配置中移除,请在参数Bucket中带上appid。Bucket由bucketname-appid组成 global secret_id, secret_key, region, token, config, client, bucket, cache_time try: secret_id = settings.COS_SECRET_ID # 替换为用户的secret_id secret_key = settings.COS_SECRET_KEY # 替换为用户的secret_key region = settings.COS_REGION # 替换为用户的region token = None # 使用临时秘钥需要传入Token,默认为空,可不填 bucket = settings.COS_BUCKET cache_time = settings.COS_CACHE_TIME # 本地缓存清理时间,以天为计,若参数出错默认为3 print("[COS]secret_id:", secret_id) print("[COS]secret_key:", secret_key) print("[COS]region:", region) print("[COS]bucket:", bucket) print("[COS]cache_time:{}".format(cache_time)) config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token) # 获取配置对象 client = CosS3Client(config) except Exception as e: log_cos.error("UnkownError:", e) print("UnkownError:", e) log_cos.info("Program Ended") sys.exit() global Main_filepath Main_filepath = main_path # print(Main_filepath) global server_mode server_mode = online_check() if server_mode: settings.COS_SERVER_MODE = 1 print("[COS]Online Server is available") log_cos.info("Cos Server is available") else: settings.COS_SERVER_MODE = 0 os.makedirs(os.path.join(settings.MEDIA_ROOT)) os.makedirs(os.path.join(settings.MEDIA_ROOT, "faces_data"), exist_ok=True) os.makedirs(os.path.join(settings.MEDIA_ROOT, "tmp"), exist_ok=True) print("[COS]Online Server is unavailable , using local storage server") log_cos.warning("Cos is unavailable , using local storage mode") # todo 接下来将对server_mode进行情况处理 # # 初始化缓存清理 schedule # global scheduler # jobstores = { # 'redis': RedisJobStore(0, host="localhost", port=6379, password=""), # 'default': SQLAlchemyJobStore(url='sqlite:///data/jobs.sqlite') # } # job_defaults = { # 'coalesce': False, # 'max_instances': 3 # } # # BackgroundScheduler: 适合于要求任何在程序后台运行的情况,当希望调度器在应用后台执行时使用。 # scheduler = BackgroundScheduler(jobstores=jobstores, job_defaults=job_defaults, daemonic=False) # scheduler.remove_all_jobs() # scheduler.add_job(Auto_DelLocalCache, 'interval', days=cache_time) # ### weeks,days,hours,minutes,seconds,start_date,end_date,timezone,jitter(最多提前或延迟作业几秒钟) # scheduler.start() # print("[COS]Start Background schedule") log_cos.info("Module COS loaded")
""" from qcloud_cos import CosConfig from qcloud_cos import CosS3Client import sys import json import logging import re from django.conf import settings ''' the client of cos ''' logging.basicConfig(level=logging.INFO, stream=sys.stdout) config = CosConfig(Region=settings.REGION, SecretId=settings.SECRETID, SecretKey=settings.SECRETKEY, Token=settings.TOKEN, Scheme=settings.SCHEMA) # 2. 获取客户端对象 client = CosS3Client(config) ''' function:upload file to Tencent cos parameters@file:the object of file@file_name:the upload path and file name of file return@str:the url of file data:2018/09/06 author:Ricky ''' def up(file, file_name): response = client.put_object(Bucket='newsapp-1254170634',
#coding:utf-8 from qcloud_cos import CosConfig from django.conf import settings from qcloud_cos import CosS3Client import sys import logging logging.basicConfig(level=logging.INFO, stream=sys.stdout) region = 'ap-nanjing' # 替换为用户的 Region 区域 config = CosConfig(Region=region, SecretId=settings.TENCENT_SECRET_ID, SecretKey=settings.TENCENT_SECRET_KEY) client = CosS3Client(config) def create_bucket(name, region): client.create_bucket( Bucket="{}-{}".format(name, settings.TENCENT_SECRET_NUMBER)) return "{}-{}".format(name, settings.TENCENT_SECRET_NUMBER) def upload_file(bucketName, file_object, key): response = client.upload_file_from_buffer( Bucket=bucketName, Body=file_object, Key=key, #上传之后的名称 )
def get_connection(store_address: dict): store_address = deepcopy(store_address) del store_address['storage'], store_address['Bucket'] return CosS3Client(CosConfig(**store_address))
import requests from qcloud_cos import CosConfig, CosS3Client from config import cos_config config = CosConfig(Region=cos_config['Region'], SecretId=cos_config['SecretId'], SecretKey=cos_config['SecretKey']) client = CosS3Client(config) def all_bucket_detail(): try: response = client.list_buckets() return response except: return None def create_bucket(bucket_name): try: client.create_bucket(Bucket=f'{bucket_name}-{cos_config["Id"]}') return True except: return False # {'Key': '1620533083236.jpg', # 'LastModified': '2021-05-09T10:38:41.000Z', # 'ETag': '"a65ec2be1c8d3f901c8fb37260fcb9c9"',
def __init__(self): config = CosConfig(Region=self.region, SecretId=self.secret_id, SecretKey=self.secret_key) self.client = CosS3Client(config)
# 而如果内容发生了变化,使用ETag有助于防止资源的同时更新相互覆盖 import os import sys import cv2 as cv import numpy as np from qcloud_cos import CosConfig from qcloud_cos import CosS3Client from qcloud_cos import CosServiceError from qcloud_cos import CosClientError from fddeye.tools import logger from io import BytesIO from fddeye.config import * try: config = CosConfig(Region=COS_REGION, SecretId=COS_SECRET_ID, SecretKey=COS_SECRET_KEY, Token=COS_TOKEN) cos_client = CosS3Client(config) except CosClientError as e: logger.error(e) def load_file(file_name,local=False): if local: load_local_file(file_name) else: with open(file_name, 'rb') as fp: response = cos_client.put_object( Bucket=COS_BUCKET, Body=fp, Key=file_name,
def __init__(self): uc = UserConfig() self._config = CosConfig(Secret_id=uc.secret_id, Secret_key=uc.secret_key, Region=uc.region, Appid=uc.appid) self._client = CosS3Client(self._config)
webhook_url = "https://app.qun.qq.com/cgi-bin/api/hookrobot_send" mirror_url = "https://sm.ms/api/v2/upload" suo_url = "http://m.6du.in/api_mobileweb_json.php" # qq群webhook key Key = os.getenv('QQHook_Key') # smms key API_token = os.getenv('smms_Token') # 腾讯云cos Secret_id = os.getenv('TX_ID') Secret_key = os.getenv('TX_Key') Region = os.getenv('TX_Region') Bucket = os.getenv('Tx_Bucket') header = {"Authorization": API_token} config = CosConfig(Region=Region, SecretId=Secret_id, SecretKey=Secret_key) client = CosS3Client(config) def SendtoQQ(name, text): print("%s:%s" % (name, text)) json_data = {"content": [{"type": 0, "data": ""}]} json_data["content"][0]["data"] = "%s:\n%s" % ( name, text) url_params = {'key': Key} with requests.post( webhook_url, json=json_data, params=url_params) as respone: # 不是很清楚原因但是,目前成功发送的http状态码确实是500,200OK反而是失败 if respone.status_code != 500: print(respone.reason)
def auth(self): config = CosConfig(Secret_id=self.cloud['SecretId'], Secret_key=self.cloud['SecretKey'], Region=self.cloud['Region']) self.client = CosS3Client(config)
config_y = yaml.load(c) FILE_RGX = config_y['file_rgx'] DATA_JSON_PATH = config_y['data_json_path'] logging.debug(FILE_RGX) # 读取上次的 Json 文件 with open(DATA_JSON_PATH, "r", encoding="utf-8") as c: MODPACK_DATA_OLD = json.load(c) logging.debug(MODPACK_DATA_OLD) # 从环境变量中获取相关数据 SECRET_ID = os.environ.get("Secret_id") SECRET_KEY = os.environ.get("Secret_key") # 登陆腾讯云 config = CosConfig(Secret_id=SECRET_ID, Secret_key=SECRET_KEY, Region='ap-guangzhou') # 获取客户端对象 client = CosS3Client(config) ################# # 相关数据的创建 # ################# # 存储整合数据 MODPACK_DATA_TOTAL = [] # 临时文件夹生成,很遗憾,生成的并不直接是路径 # 而且不允许直接带 .name 获取字符串 TMP_WIN_DIR = tempfile.TemporaryDirectory() TMP_LINUX_DIR = tempfile.TemporaryDirectory()
def Initialize(cfg_path: str, main_path: str): """ COS 模块初始化,此函数应在所有函数之前调用 :param cfg_path: 配置文件地址。 :param main_path: 程序主目录地址。 """ global cf cf = configparser.ConfigParser() cf.read(cfg_path) # 设置用户属性, 包括secret_id, secret_key, region # appid已在配置中移除,请在参数Bucket中带上appid。Bucket由bucketname-appid组成 global secret_id, secret_key, region, token, config, client, bucket, cache_time try: secret_id = str(cf.get("COS", "secret_id")) # 替换为用户的secret_id secret_key = str(cf.get("COS", "secret_key")) # 替换为用户的secret_key region = str(cf.get("COS", "region")) # 替换为用户的region token = None # 使用临时秘钥需要传入Token,默认为空,可不填 bucket = str(cf.get("COS", "bucket")) cache_time_str = str(cf.get("COS", "cache_time")) # 本地缓存清理时间,以天为计,若参数出错默认为3 if cache_time_str.isdigit(): cache_time = int(cache_time_str) else: cache_time = 3 print("[COS]secret_id:", secret_id) print("[COS]secret_key:", secret_key) print("[COS]region:", region) print("[COS]bucket:", bucket) print("[COS]cache_time:{}".format(cache_time)) config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token) # 获取配置对象 client = CosS3Client(config) except Exception as e: log_cos.error("UnkownError:", e) print("UnkownError:", e) log_cos.info("Program Ended") sys.exit() global Main_filepath sysstr = platform.system() if sysstr == "Windows": print("[COS]System: Windows") Main_filepath = main_path elif sysstr == "Linux": print("[COS]System: Linux") Main_filepath = "/tmp" # print(Main_filepath) else: Main_filepath = main_path global server_mode server_mode = online_check() if server_mode: cf.set("COS", "server_mode", "1") print("[COS]Online Server is available") log_cos.info("Cos Server is available") else: cf.set("COS", "server_mode", "0") os.makedirs(os.path.join(Main_filepath, "data", "local", "portrait"), exist_ok=True) os.makedirs(os.path.join(Main_filepath, "data", "local", "article"), exist_ok=True) print("[COS]Online Server is unavailable , using local storage server") log_cos.warning("Cos is unavailable , using local storage mode") # 初始化缓存清理 schedule global scheduler if sysstr == "Windows": jobstores = { 'redis': RedisJobStore(0, host="localhost", port=6379, password=""), 'default': SQLAlchemyJobStore(url='sqlite:///data/jobs.sqlite') } elif sysstr == "Linux": jobstores = { 'redis': RedisJobStore(0, host="localhost", port=6379, password=""), 'default': SQLAlchemyJobStore(url='sqlite:///tmp/data/jobs.sqlite') } else: jobstores = { 'redis': RedisJobStore(0, host="localhost", port=6379, password=""), 'default': SQLAlchemyJobStore(url='sqlite:///data/jobs.sqlite') } job_defaults = {'coalesce': False, 'max_instances': 3} # BackgroundScheduler: 适合于要求任何在程序后台运行的情况,当希望调度器在应用后台执行时使用。 scheduler = BackgroundScheduler(jobstores=jobstores, job_defaults=job_defaults, daemonic=False) scheduler.remove_all_jobs() scheduler.add_job(Auto_DelLocalCache, 'interval', days=cache_time) ### weeks,days,hours,minutes,seconds,start_date,end_date,timezone,jitter(最多提前或延迟作业几秒钟) scheduler.start() print("[COS]Start Background schedule") log_cos.info("Module COS loaded")
import numpy as np import cv2 import tempfile import uuid import traceback import yaml import time from qcloud_cos import CosConfig from qcloud_cos import CosS3Client DEBUG = False cos = yaml.load(open('cos.yaml')) bucket = cos['bucket'] config = CosConfig(Region=cos['region'], SecretId=cos['secret_id'], SecretKey=cos['secret_key'], Scheme='https') client = CosS3Client(config) cache_dir = cos['cache_dir'] def to_image_key(image_file, image_key_or_prefix): if image_key_or_prefix.endswith('_'): image_prefix = image_key_or_prefix image_key = image_prefix + str(uuid.uuid4()) else: image_key = image_key_or_prefix if DEBUG: print('New image key: {}\n{}'.format(image_key, ''.join(traceback.format_stack()))) with open(image_file, 'r') as f: client.put_object( Bucket=bucket,
pip install -U cos-python-sdk-v5 $shortcut = (New-Object -COM WScript.Shell).CreateShortcut("$($env:APPDATA)\Microsoft\Windows\SendTo\Tencent COS.lnk") $shortcut.TargetPath = (Get-Command -Name python | Select-Object -Property Source -First 1).Source $shortcut.Arguments = (Get-Command -Name upload-to-cos | Select-Object -Property Source -First 1).Source $shortcut.Save() [Usage] Right click a file, then navigate to "send to", then click "COS". """ AK = "<Tencent Cloud COS SecretId>" SK = "<Tencent Cloud COS SecretKey>" REGION = "ap-shanghai" BUCKET = "<Bucket Id>-<AppId>" # something like "cdn-123456" PREFIX = "by-uuid/" client = CosS3Client(CosConfig(Region=REGION, SecretId=AK, SecretKey=SK)) files = sys.argv[1:] for file in files: print("\r\n" + "-" * 40 + "\r\n") if os.path.isdir(file): print("Skipping Folder:", file) continue print("Uploading File:", file) extname = file.split(".")[-1] with open(file, "rb") as f: target_name = "{}.{}".format(str(uuid.uuid4()), extname) response = client.put_object(Bucket=BUCKET, Body=f, Key=PREFIX + target_name, StorageClass='STANDARD', EnableMD5=False)
def upload(self, region, request): if not self.ignore_check: self._prefix_check_and_set_default_val(region, request) request_str = request.to_json_string() logger.info("vod upload req = {}, region = {}".format(request_str, region)) cred = credential.Credential(self.secret_id, self.secret_key, self.token) api_client = vod_client.VodClient(cred, region) parsed_manifest_list = [] segment_file_path_list = [] if self._is_manifest_media_tpe(request.MediaType): self.parse_manifest(api_client, request.MediaFilePath, request.MediaType, parsed_manifest_list, segment_file_path_list) apply_upload_request = models.ApplyUploadRequest() apply_upload_request.from_json_string(request_str) apply_upload_response = self.apply_upload(api_client, apply_upload_request) logger.info("vod upload ApplyUpload rsp = {}".format(apply_upload_response.to_json_string())) if apply_upload_response.TempCertificate is None: cos_config = CosConfig( Region=apply_upload_response.StorageRegion, SecretId=self.secret_id, SecretKey=self.secret_key ) else: temp_certificate = apply_upload_response.TempCertificate cos_config = CosConfig( Region=apply_upload_response.StorageRegion, SecretId=temp_certificate.SecretId, SecretKey=temp_certificate.SecretKey, Token=temp_certificate.Token ) cos_client = CosS3Client(cos_config) if StringUtil.is_not_empty(request.MediaType) \ and StringUtil.is_not_empty(apply_upload_response.MediaStoragePath): self.upload_cos( cos_client, request.MediaFilePath, apply_upload_response.StorageBucket, apply_upload_response.MediaStoragePath[1:], request.ConcurrentUploadNumber ) if StringUtil.is_not_empty(request.CoverType) \ and StringUtil.is_not_empty(apply_upload_response.CoverStoragePath): self.upload_cos( cos_client, request.CoverFilePath, apply_upload_response.StorageBucket, apply_upload_response.CoverStoragePath[1:], request.ConcurrentUploadNumber ) if len(segment_file_path_list) > 0: for segment_file_path in segment_file_path_list: storage_dir = os.path.dirname(apply_upload_response.MediaStoragePath) media_file_dir = os.path.dirname(request.MediaFilePath) segment_relative_file_path = segment_file_path[len(media_file_dir):].replace("\\", "/") segment_storage_path = FileUtil.join_path(storage_dir, segment_relative_file_path) self.upload_cos( cos_client, segment_file_path, apply_upload_response.StorageBucket, segment_storage_path[1:], request.ConcurrentUploadNumber ) commit_upload_request = models.CommitUploadRequest() commit_upload_request.VodSessionKey = apply_upload_response.VodSessionKey commit_upload_request.SubAppId = request.SubAppId commit_upload_response = self.commit_upload(api_client, commit_upload_request) commit_upload_response_str = commit_upload_response.to_json_string() logger.info("vod upload CommitUpload rsp = {}".format(commit_upload_response_str)) response = VodUploadResponse() response.from_json_string(commit_upload_response_str) return response
import os from qcloud_cos import CosConfig from qcloud_cos import CosS3Client from ti.utils import get_temporary_secret_and_token ''' 上传embedding文件到tone平台 ''' #### 指定本地文件路径,可根据需要修改。 local_file = "/home/tione/notebook/embedding/embedding_diff.zip" #### 用户的存储桶,修改为存放所需数据文件的存储桶,存储桶获取参考腾讯云对象存储 bucket="..." #### 用户的数据,修改为对应的数据文件路径,文件路径获取参考腾讯云对象存储 data_key="embedding_diff/embedding_diff.zip" #### 获取用户临时密钥 secret_id, secret_key, token = get_temporary_secret_and_token() config = CosConfig(Region=os.environ.get('REGION'), SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme='https') client = CosS3Client(config) #### 获取文件到本地 response = client.get_object( Bucket=bucket, Key=data_key, ) response['Body'].get_stream_to_file(local_file) print('finish.....')
from qcloud_image import Client from qcloud_image import CIUrl, CIFile, CIBuffer, CIUrls, CIFiles, CIBuffers from PIL import Image import PIL.Image import sys logging.basicConfig(level=logging.INFO, stream=sys.stdout) appid = 1251762227 # 请替换为您的 APPID secret_id = u'****' # 请替换为您的 SecretId secret_key = u'****' # 请替换为您的 SecretKey region = u'ap-beijing' token = '' config = CosConfig(Secret_id=secret_id, Secret_key=secret_key, Region=region, Token=token) client = CosS3Client(config) logger = logging.getLogger() def resize_image(image_path, resized_path): with Image.open(image_path) as image: size = float(os.path.getsize(image_path) / 1024 / 1024) print size if size >= 1: image.thumbnail(tuple(x / 2 for x in image.size)) # image.resize((1024,1024),Image.BILINEAR) image.save(resized_path)
def update(self, *args, **kwargs): # 支持WEBDAV, MLTALKER # antvis/experiment_uuid/group/%s_(latest_%d).xxx if len(args) == 0: return local_file_path = os.path.normpath(args[0]) if not os.path.exists(local_file_path): return # 扩展名 ext_name = local_file_path.split('.')[-1] # 文件名 file_name = local_file_path.split('/')[-1] # 单位M fsize = os.path.getsize(local_file_path) fsize = fsize / float(1024 * 1024) fsize = round(fsize, 2) # 1.step 获得临时秘钥 response = \ mlogger.getEnv().dashboard.rpc.cos.experiment.get(file_size=fsize, operator='upload') if response['status'] == 'ERROR': logging.error('could get cos credential') return tmpSecretId = response['content']['tmpSecretId'] tmpSecretKey = response['content']['tmpSecretKey'] sessionToken = response['content']['sessionToken'] region = response['content']['region'] bucket = response['content']['bucket'] # 2.step 上传文件 tag_str = self.tag if self.group_name == '' else '%s/%s' % ( self.group_name, self.tag) try: config = CosConfig(Region=region, SecretId=tmpSecretId, SecretKey=tmpSecretKey, Token=sessionToken, Scheme='https') client = CosS3Client(config) with open(local_file_path, 'rb') as fp: client.put_object(Bucket=bucket, Body=fp, Key='{}/{}/{}/{}'.format( self.experiment_uuid, tag_str, self.title, file_name), StorageClass='STANDARD', EnableMD5=False) except: logging.error('upload {} error'.format(local_file_path)) return # 3.step 更新平台记录 mlogger.getEnv().dashboard.experiment.patch( **{ 'experiment_name': self.experiment_name, 'experiment_uuid': self.experiment_uuid, 'experiment_stage': mlogger.getEnv().dashboard.experiment_stage, 'experiment_data': json.dumps({ 'FILE_ABSTRACT': { 'group': tag_str, 'title': self.title, 'backend': 'ANTVIS', 'path': '{}/{}/{}/{}'.format(self.experiment_uuid, tag_str, self.title, file_name), 'size': fsize, }, 'APP_STAGE': mlogger.getEnv().dashboard.experiment_stage }) })
def getPageHtml(self) -> str: r = requests.post( 'http://10.10.10.2:3001/function', data=json.dumps({ 'context': { 'url': self.url }, 'code': "module.exports=async({page,context:b})=>{const{url}=b;await page.goto(url,{waitUntil:'networkidle2'});const data=await page.evaluate(()=>{var html=document.getElementsByClassName('notion-page-content')[0].innerHTML;var frag=document.createElement('div');frag.innerHTML=html;var imglist=[].map.call(frag.querySelectorAll('img'),function(img){return img.src});return{html:frag.innerHTML,imglist}});return{data:data,type:'application/json'}};" }), headers={'Content-Type': 'application/json'}) if r.status_code != 200: return "error" need_del_dict = { '/image/https': 'https://www.notion.so/image/https', 'href="/': 'href="https://www.notion.so/', '/images/emoji': 'https://www.notion.so/images/emoji' } pattern = re.compile(r'max-width.*?(?:;)') json_response = r.content.decode() json_dict = json.loads(json_response) content = json_dict['html'] need_del_list = pattern.findall(content) need_del_list = list(dict.fromkeys(need_del_list)) need_del_list.remove('max-width: 100%;') for item in need_del_list: need_del_dict[item] = '' for key in need_del_dict: content = content.replace(key, need_del_dict[key]) img_pattern = re.compile( r'https:\/\/www\.notion\.so\/image\/https.*?(?:;cache=v2)') img_url_list = img_pattern.findall(content) for img_url in img_url_list: secret_id = os.getenv('SECRET_ID') secret_key = os.getenv('SECRET_KEY') region = os.getenv('REGION') bucket = os.getenv('BUCKET') base_url = os.getenv('CDNURL') token = None scheme = 'https' config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) client = CosS3Client(config) r = requests.get(img_url.replace("amp;", "")) print(img_url.replace("amp;", "")) print(img_url.replace("amp;", "")) if r.headers['Content-Type'].find('image') == -1: continue file_name = r.headers['Content-Type'] + "/" + str(int( time.time())) + "." + r.headers['Content-Type'].split('/')[-1] file_url = f"{base_url}{file_name}" response = client.put_object( Bucket=bucket, Body=r.content, Key=file_name, ) print(response) print(file_url) content = content.replace(img_url, file_url) return content
) print r def get(client, file_name): r = client.get_object( Bucket=bucket, Key=file_name, ) r['Body'].get_stream_to_file(file_name) # r['Body'].get_raw_stream() 流 if __name__ == '__main__': config = CosConfig(Secret_id=secret_id, Secret_key=secret_key, Region=region) client = CosS3Client(conf=config) try: if not os.path.isdir(temp_dir): os.mkdir(temp_dir) except IOError, err: print err sys.exit() full_temp = os.path.join(temp_dir, backup_filename) tar = tarfile.open(name=full_temp, mode='w:gz') for dirpath, dirnames, filenames in os.walk(backup_dir): for file in filenames: full_path = os.path.join(dirpath, file)
from qcloud_cos.crypto import RSAProvider from qcloud_cos.cos_comm import CiDetectType SECRET_ID = os.environ["SECRET_ID"] SECRET_KEY = os.environ["SECRET_KEY"] TRAVIS_FLAG = os.environ["TRAVIS_FLAG"] REGION = os.environ["REGION"] APPID = '1251668577' test_bucket = 'cos-python-v5-test-' + str(sys.version_info[0]) + '-' + str( sys.version_info[1]) + '-' + REGION + '-' + APPID copy_test_bucket = 'copy-' + test_bucket test_object = "test.txt" special_file_name = "中文" + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" conf = CosConfig( Region=REGION, SecretId=SECRET_ID, SecretKey=SECRET_KEY, ) client = CosS3Client(conf, retry=3) rsa_provider = RSAProvider() client_for_rsa = CosEncryptionClient(conf, rsa_provider) aes_provider = AESProvider() client_for_aes = CosEncryptionClient(conf, aes_provider) def _create_test_bucket(test_bucket, create_region=None): try: if create_region is None: response = client.create_bucket(Bucket=test_bucket, ) else: bucket_conf = CosConfig(Region=create_region,
def sync(self): # 本地文件目录列表 local_file_dir_info = LocalFileDirInfo(self.config.get_local_path()) # 记录的要删除的文件和目录列表 del_file_dict = self.db.file_stat_dict del_dir_dict = self.db.dir_stat_dict # 记录的要创建的文件和目录列表 create_file_dict = {} create_dir_dict = {} # 文件同步状态校验 for local_file_stat in local_file_dir_info.file_stat_list: local_file_path = local_file_stat.file_path # 如果数据库中记录cos中也存在, 则判断下文件是否发生了改变 if (self.db.file_stat_dict.has_key(local_file_path)): # 如果文件内容未发生变化, 则不用同步, 否则进行同步 if (FileStat.compare( local_file_stat, self.db.file_stat_dict[local_file_path]) == 0): del self.db.file_stat_dict[local_file_path] else: create_file_dict[local_file_path] = local_file_stat # 如果数据库中不存在相关记录, 则插入到要创建的文件列表 else: create_file_dict[local_file_path] = local_file_stat # 目录同步状态校验 for local_dir_stat in local_file_dir_info.dir_stat_list: local_dir_path = local_dir_stat.file_path # 如果数据库中记录cos中也存在, 则判断下目录是否发生了改变 if (self.db.dir_stat_dict.has_key(local_dir_path)): del self.db.dir_stat_dict[local_dir_path] # 如果数据库中不存在相关记录, 则插入到要创建的文件列表 else: create_dir_dict[local_dir_path] = local_dir_stat # 生成cos客户端 appid = int(self.config.get_appid()) secret_id = self.config.get_secret_id() secret_key = self.config.get_secret_key() bucket = self.config.get_bucket() timeout = self.config.get_timeout() cos_client = CosClient(appid, secret_id, secret_key) cos_config = CosConfig() cos_config.set_timeout(timeout) if self.config.get_enable_https() == 1: cos_config.enable_https() cos_client.set_config(cos_config) # 创建要同步的cos 目录 self._create_cos_target_dir(cos_client, bucket) # 进行同步, 同步顺序如下 # 1 删除文件(之前本地有,现在没有的文件, 或者发生修改的文件) self._del_cos_sync_file(cos_client, bucket, del_file_dict) # 2 删除目录(之前本地有,现在没有的目录) self._del_cos_sync_dir(cos_client, bucket, del_dir_dict) # 3 创建目录(新建的目录) self._create_cos_sync_dir(cos_client, bucket, create_dir_dict) # 4 创建文件(新建的文件) self._create_cos_sync_file(cos_client, bucket, create_file_dict)
#!/usr/bin/env python # -*- coding:utf-8 -*- from qcloud_cos import CosConfig from qcloud_cos import CosS3Client secret_id = 'XXXXXXXXXXXXXXXX' # 替换为用户的 secretId secret_key = 'XXXXXXXXXXXXXXXXXXX' # 替换为用户的 secretKey region = 'ap-hangzhou' # 替换为用户的 Region config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key) client = CosS3Client(config) response = client.create_bucket( Bucket='test-1251317460', ACL="public-read" # private / public-read / public-read-write )
# -*- coding=utf-8 from qcloud_cos import CosConfig from qcloud_cos import CosS3Client import time import sys import logging from GameNews.settings import COS_SECRETKEY, COS_SECRETID, COS_REGION, COS_BUCKET logging.basicConfig(level=logging.INFO, stream=sys.stdout) token = None # 使用临时密钥需要传入Token,默认为空,可不填 config = CosConfig(Region=COS_REGION, SecretId=COS_SECRETID, SecretKey=COS_SECRETKEY, Token=token) # 获取配置对象 client = CosS3Client(config) def upload_img_byte_stream(file, file_name): """ 字节流上传文件 """ ts = time.time() response = client.put_object(Bucket=COS_BUCKET, Body=file, Key='image' + str(ts) + file_name) return response['ETag']