def open_spider(self, spider): self.config= Config(self.qs_asscess_key,self.qs_secret_key) self.qingstor = QingStor(self.config) self.bucket = self.qingstor.Bucket(self.qs_bucket_name,self.qs_zone) self.session = requests.Session()
def _init_bucket(self, bucket_name, bucket_zone): qingstor = QingStor(self.config) bucket = qingstor.Bucket(bucket_name, bucket_zone) list_bucket_output = qingstor.list_buckets() bucket_names = [] for bucket_exist in list_bucket_output['buckets']: bucket_names.append(bucket_exist['name']) if bucket_name not in bucket_names: bucket.put() return bucket
def __init__(self, access_key_id=None, secret_access_key=None, zone=None, bucket_name=None): self.access_key_id = access_key_id if access_key_id else _get_config( 'QINGSTOR_ACCESS_KEY_ID') self.secret_access_key = secret_access_key if secret_access_key else _get_config( 'QINGSTOR_SECRET_ACCESS_KEY') self.zone = zone if zone else _get_config('QINGSTOR_ZONE') self.bucket_name = bucket_name if bucket_name else _get_config( 'QINGSTOR_BUCKET') # get bucket config = Config(self.access_key_id, self.secret_access_key) qingstor = QingStor(config) self.bucket = qingstor.Bucket(bucket_name=self.bucket_name, zone=self.zone)
def get_connection(self, conf): try: key_id = conf.qy_access_key_id except AttributeError: print('[ERROR] cannot find key id info in config file') sys.exit(-1) try: secret_key = conf.qy_secret_access_key except AttributeError: print('[ERROR] cannot find secret key info in config file') sys.exit(-1) try: zone = conf.zone except AttributeError: print('[ERROR] cannot find zone info in config file') sys.exit(-1) config = Config(key_id, secret_key) service = QingStor(config) return service
from qingstor.sdk.service.qingstor import QingStor from qingstor.sdk.config import Config ACCESS_KEY_ID = 'KPXLUFSRVNVNZGFCEPDT' SECRET_ACCESS_KEY = '9RW7JW2RsIDmArXSdeHhCjYt7A9vHPs6LBT8zSEp' config = Config(ACCESS_KEY_ID, SECRET_ACCESS_KEY) qingstor = QingStor(config) # List all buckets. output = qingstor.list_buckets() # Print HTTP status code. print(output.status_code) # Print the buckets. print(output['buckets']) bucket = qingstor.Bucket('mrc-lxm', 'pek3b') fname = 'main1.jpg' with open('trendsetter/img/{}'.format(fname), 'rb') as f: output = bucket.put_object(fname, body=f) # Print the HTTP status code. # Example: 201 print(output.status_code)
import json from os import path import yaml from assertpy import assert_that from behave import * from qingstor.sdk.config import Config from qingstor.sdk.service.qingstor import QingStor test_config_file_path = path.abspath( path.join(path.dirname(__file__), path.pardir)) config = Config().load_config_from_filepath(test_config_file_path + "/config.yaml") qingstor = QingStor(config) with open(test_config_file_path + '/test_config.yaml') as f: test = yaml.load(f) f.close() bucket = qingstor.Bucket(test['bucket_name'], test['zone']) bucket.put() @when(u'put bucket ACL') def step_impl(context): context.res = bucket.put_acl(json.loads(context.text)['acl']) @then(u'put bucket ACL status code is 200') def step_impl(context): assert_that(context.res.status_code).is_equal_to(200)
def add_assets(username, type, music_name): # 配置青云服务器 config = Config(qy_access_key, qy_secret_access_key) service = QingStor(config) buckets = service.list_buckets('pek3a')['buckets'] print(buckets) # 获取业务用 bucket [bucket_info] = filter(lambda bucket: bucket['name'] == 'nest-habit', buckets) bucket = service.Bucket('nest-habit', 'pek3a') # 随机一个文件名 if type == 'uploaded_musics': base = 'music' elif type == 'avatar': base = 'image' else: return json.dumps({'error': 'Invalid operation!'}), 400, regular_req_headers filename = mimetypes.guess_extension(request.headers.get('x-mime-type')) filename = '%s/%s%s' % (base, str(random.randint(1, 1E15)), filename) # 上传这个文件 result = bucket.put_object(filename, body=request.data) print(result) # 组装 URL url = bucket_info['url'] + '/' + filename if type == 'uploaded_musics': # 更新数据库 inserted_id = db['_musics'].insert_one({ 'url': url, 'music_name': music_name, 'created_time': datetime.datetime.utcnow(), 'creator': username }).inserted_id result = db['_users'].find_one_and_update( {'username': username}, {'$push': { 'uploaded_musics': inserted_id }}, return_document=ReturnDocument.AFTER) elif type == 'avatar': result = db['_users'].find_one_and_update( {'username': username}, {'$set': { 'avatar': url }}, return_document=ReturnDocument.AFTER) else: return json.dumps({'error': 'Invalid operation!'}), 400, regular_req_headers if result == None: bucket.delete_object(filename) return _no_user_named_xxx, 400, regular_req_headers # 生成响应 if type == 'uploaded_musics': result = {'_id': inserted_id, 'music_name': music_name, 'url': url} return json.dumps(result, default=oid_handler), 200, regular_req_headers keys = list( filter(lambda key: key not in ['_id', 'password', 'created_time'], result.keys())) values = list(map(lambda key: result[key], keys)) return json.dumps(dict(zip(keys, values)), default=oid_handler), 200, regular_req_headers
def get_client(cls, conf): config = Config().load_config_from_data(conf) return QingStor(config)
class MirrorhttprsynctoolPipeline(object): TOO_LONG = 1024*1024*64 def __init__(self,qs_access_key,qs_secret_key,qs_zone,qs_bucket_name,qs_bucket_prefix): self.qs_asscess_key = qs_access_key self.qs_secret_key = qs_secret_key self.qs_zone = qs_zone self.qs_bucket_name = qs_bucket_name self.qs_bucket_prefix = qs_bucket_prefix @classmethod def from_crawler(cls, crawler): return cls( qs_access_key = crawler.settings.get("QS_ACCESS_KEY"), qs_secret_key = crawler.settings.get("QS_SECRET_KEY"), qs_zone = crawler.settings.get("QS_ZONE"), qs_bucket_name = crawler.settings.get("QS_BUCKET_NAME"), qs_bucket_prefix = crawler.settings.get("QS_BUCKET_PREFIX") ) def process_item(self, item, spider): url= item['url'] key= '' # remove / in prefix if self.qs_bucket_prefix is not None and len(self.qs_bucket_prefix) >0: key = self.qs_bucket_prefix+urlparse(url).path else : key = urlparse(url).path[1:] if not self.is_timestamp_equal(url,key): logging.info("%s updated. download new version from repo",key) try: with self.session.get(url, stream=True) as response: response.raise_for_status() if int(response.headers['content-length']) < self.TOO_LONG: self.bucket.put_object(key,body=response.content) logging.log(logging.INFO,"upload %s" % key) else: uploadJobinfo=self.bucket.initiate_multipart_upload(key) uploadid= uploadJobinfo['upload_id'] partNo = 0 partList= [] # Throw an error for bad status codes for block in response.iter_content(chunk_size=self.TOO_LONG): output = self.bucket.upload_multipart(key,upload_id=uploadid,part_number=partNo,body=block) logging.info("upload: %s,id:%s,part %d,status: %d" ,key,uploadid,partNo,output.status_code) partNo +=1 partList.append(partNo) self.bucket.complete_multipart_upload(key,upload_id=uploadid,object_parts=[{"part_number":x} for x in partList]) logging.info("uploaded: %s" ,key) except: logging.exception("Failed to download %s Unexpected error.",url) return item def is_timestamp_equal(self,url,key): objStatus=self.bucket.head_object(key) if objStatus.status_code == 200 : with self.session.head(url) as response: logging.info("key: %s source length:%s target length %s modified %s source %s",key,response.headers['Content-Length'],objStatus.headers['Content-Length'],response.headers['Last-Modified'],objStatus.headers['Last-Modified']) return objStatus.headers['Content-Length'] == response.headers['Content-Length'] #and objStatus.headers['Last-Modified']== response.headers['Last-Modified'] return False def open_spider(self, spider): self.config= Config(self.qs_asscess_key,self.qs_secret_key) self.qingstor = QingStor(self.config) self.bucket = self.qingstor.Bucket(self.qs_bucket_name,self.qs_zone) self.session = requests.Session()
# -*- coding: utf-8 -*- from __future__ import unicode_literals import os import sh import yaml from behave import * from assertpy import assert_that from qingstor.sdk.config import Config from qingstor.sdk.service.qingstor import QingStor config = Config().load_user_config() qingstor = QingStor(config) test_config_file_path = os.path.abspath( os.path.join(os.path.dirname(__file__), os.path.pardir)) with open(test_config_file_path + '/test_config.yaml') as f: test_data = yaml.load(f) f.close() bucket = qingstor.Bucket(test_data["bucket_name"] + "1", test_data["zone"]) qsctl = sh.Command("qsctl") @when(u'create a bucket') def step_impl(context): qsctl("mb", "qs://{bucket}".format(bucket=test_data["bucket_name"] + "1")) @then(u'should get a new bucket')
def step_impl(context): context.qingstor = QingStor(config) context.res = context.qingstor.list_buckets()
def step_impl(context): context.qingstor = QingStor(config)
# -*- coding:utf-8 -*- from qingstor.sdk.service.qingstor import QingStor from qingstor.sdk.config import Config import requests import tempfile # 初始化QingStor对象 config = Config('EBZAULNLJTMJJCZJWCRG', '02MLUQZA5wZPwTrOmVWUajPhNUSFI8c5VKu2Wf0u') qingstor = QingStor(config) # 获取bucket列表 # output = qingstor.list_buckets() # print(output.status_code) # print(output['buckets']) # 获取指定bucket对象 bucket = qingstor.Bucket('fifa-helper-bucket', 'sh1a') def getFileByKey(key): resp = bucket.get_object(key) with tempfile.NamedTemporaryFile() as f: for chunk in resp.iter_content(): f.write(chunk) return f def getRespByKey(key): resp = bucket.get_object(key)
# -- bucket -- from qingstor.sdk.service.qingstor import QingStor from qingstor.sdk.config import Config ACCESS_KEY_ID = 'KPXLUFSRVNVNZGFCEPDT' SECRET_ACCESS_KEY = '9RW7JW2RsIDmArXSdeHhCjYt7A9vHPs6LBT8zSEp' prefix = 'https://mrc-lxm.pek3b.qingstor.com/' tmp_path = '/tmp/mrc' if sys.platform == 'win32': tmp_path = os.path.join(os.environ['TMP'], 'mrc') config = Config(ACCESS_KEY_ID, SECRET_ACCESS_KEY) qingstor = QingStor(config) bucket = qingstor.Bucket('mrc-lxm', 'pek3b') # -- model -- import torch from fairseq.data.data_utils import collate_tokens from fairseq.models.roberta import RobertaModel # -- translate -- from translate import translate # -- utils -- def clear_unascii(s): return ''.join([c for c in s if ascii.isascii(c)])