def open_spider(self, spider):
self.config= Config(self.qs_asscess_key,self.qs_secret_key)
self.qingstor = QingStor(self.config)
self.bucket = self.qingstor.Bucket(self.qs_bucket_name,self.qs_zone)
self.session = requests.Session()
def _init_bucket(self, bucket_name, bucket_zone):
qingstor = QingStor(self.config)
bucket = qingstor.Bucket(bucket_name, bucket_zone)
list_bucket_output = qingstor.list_buckets()
bucket_names = []
for bucket_exist in list_bucket_output['buckets']:
bucket_names.append(bucket_exist['name'])
if bucket_name not in bucket_names:
bucket.put()
return bucket
def __init__(self,
access_key_id=None,
secret_access_key=None,
zone=None,
bucket_name=None):
self.access_key_id = access_key_id if access_key_id else _get_config(
'QINGSTOR_ACCESS_KEY_ID')
self.secret_access_key = secret_access_key if secret_access_key else _get_config(
'QINGSTOR_SECRET_ACCESS_KEY')
self.zone = zone if zone else _get_config('QINGSTOR_ZONE')
self.bucket_name = bucket_name if bucket_name else _get_config(
'QINGSTOR_BUCKET')
# get bucket
config = Config(self.access_key_id, self.secret_access_key)
qingstor = QingStor(config)
self.bucket = qingstor.Bucket(bucket_name=self.bucket_name,
zone=self.zone)
def get_connection(self, conf):
try:
key_id = conf.qy_access_key_id
except AttributeError:
print('[ERROR] cannot find key id info in config file')
sys.exit(-1)
try:
secret_key = conf.qy_secret_access_key
except AttributeError:
print('[ERROR] cannot find secret key info in config file')
sys.exit(-1)
try:
zone = conf.zone
except AttributeError:
print('[ERROR] cannot find zone info in config file')
sys.exit(-1)
config = Config(key_id, secret_key)
service = QingStor(config)
return service
from qingstor.sdk.service.qingstor import QingStor
from qingstor.sdk.config import Config
ACCESS_KEY_ID = 'KPXLUFSRVNVNZGFCEPDT'
SECRET_ACCESS_KEY = '9RW7JW2RsIDmArXSdeHhCjYt7A9vHPs6LBT8zSEp'
config = Config(ACCESS_KEY_ID, SECRET_ACCESS_KEY)
qingstor = QingStor(config)
# List all buckets.
output = qingstor.list_buckets()
# Print HTTP status code.
print(output.status_code)
# Print the buckets.
print(output['buckets'])
bucket = qingstor.Bucket('mrc-lxm', 'pek3b')
fname = 'main1.jpg'
with open('trendsetter/img/{}'.format(fname), 'rb') as f:
output = bucket.put_object(fname, body=f)
# Print the HTTP status code.
# Example: 201
print(output.status_code)
import json
from os import path
import yaml
from assertpy import assert_that
from behave import *
from qingstor.sdk.config import Config
from qingstor.sdk.service.qingstor import QingStor
test_config_file_path = path.abspath(
path.join(path.dirname(__file__), path.pardir))
config = Config().load_config_from_filepath(test_config_file_path +
"/config.yaml")
qingstor = QingStor(config)
with open(test_config_file_path + '/test_config.yaml') as f:
test = yaml.load(f)
f.close()
bucket = qingstor.Bucket(test['bucket_name'], test['zone'])
bucket.put()
@when(u'put bucket ACL')
def step_impl(context):
context.res = bucket.put_acl(json.loads(context.text)['acl'])
@then(u'put bucket ACL status code is 200')
def step_impl(context):
assert_that(context.res.status_code).is_equal_to(200)
def add_assets(username, type, music_name):
# 配置青云服务器
config = Config(qy_access_key, qy_secret_access_key)
service = QingStor(config)
buckets = service.list_buckets('pek3a')['buckets']
print(buckets)
# 获取业务用 bucket
[bucket_info] = filter(lambda bucket: bucket['name'] == 'nest-habit',
buckets)
bucket = service.Bucket('nest-habit', 'pek3a')
# 随机一个文件名
if type == 'uploaded_musics':
base = 'music'
elif type == 'avatar':
base = 'image'
else:
return json.dumps({'error':
'Invalid operation!'}), 400, regular_req_headers
filename = mimetypes.guess_extension(request.headers.get('x-mime-type'))
filename = '%s/%s%s' % (base, str(random.randint(1, 1E15)), filename)
# 上传这个文件
result = bucket.put_object(filename, body=request.data)
print(result)
# 组装 URL
url = bucket_info['url'] + '/' + filename
if type == 'uploaded_musics':
# 更新数据库
inserted_id = db['_musics'].insert_one({
'url':
url,
'music_name':
music_name,
'created_time':
datetime.datetime.utcnow(),
'creator':
username
}).inserted_id
result = db['_users'].find_one_and_update(
{'username': username},
{'$push': {
'uploaded_musics': inserted_id
}},
return_document=ReturnDocument.AFTER)
elif type == 'avatar':
result = db['_users'].find_one_and_update(
{'username': username}, {'$set': {
'avatar': url
}},
return_document=ReturnDocument.AFTER)
else:
return json.dumps({'error':
'Invalid operation!'}), 400, regular_req_headers
if result == None:
bucket.delete_object(filename)
return _no_user_named_xxx, 400, regular_req_headers
# 生成响应
if type == 'uploaded_musics':
result = {'_id': inserted_id, 'music_name': music_name, 'url': url}
return json.dumps(result,
default=oid_handler), 200, regular_req_headers
keys = list(
filter(lambda key: key not in ['_id', 'password', 'created_time'],
result.keys()))
values = list(map(lambda key: result[key], keys))
return json.dumps(dict(zip(keys, values)),
default=oid_handler), 200, regular_req_headers
def get_client(cls, conf):
config = Config().load_config_from_data(conf)
return QingStor(config)
class MirrorhttprsynctoolPipeline(object):
TOO_LONG = 1024*1024*64
def __init__(self,qs_access_key,qs_secret_key,qs_zone,qs_bucket_name,qs_bucket_prefix):
self.qs_asscess_key = qs_access_key
self.qs_secret_key = qs_secret_key
self.qs_zone = qs_zone
self.qs_bucket_name = qs_bucket_name
self.qs_bucket_prefix = qs_bucket_prefix
@classmethod
def from_crawler(cls, crawler):
return cls(
qs_access_key = crawler.settings.get("QS_ACCESS_KEY"),
qs_secret_key = crawler.settings.get("QS_SECRET_KEY"),
qs_zone = crawler.settings.get("QS_ZONE"),
qs_bucket_name = crawler.settings.get("QS_BUCKET_NAME"),
qs_bucket_prefix = crawler.settings.get("QS_BUCKET_PREFIX")
)
def process_item(self, item, spider):
url= item['url']
key= ''
# remove / in prefix
if self.qs_bucket_prefix is not None and len(self.qs_bucket_prefix) >0:
key = self.qs_bucket_prefix+urlparse(url).path
else :
key = urlparse(url).path[1:]
if not self.is_timestamp_equal(url,key):
logging.info("%s updated. download new version from repo",key)
try:
with self.session.get(url, stream=True) as response:
response.raise_for_status()
if int(response.headers['content-length']) < self.TOO_LONG:
self.bucket.put_object(key,body=response.content)
logging.log(logging.INFO,"upload %s" % key)
else:
uploadJobinfo=self.bucket.initiate_multipart_upload(key)
uploadid= uploadJobinfo['upload_id']
partNo = 0
partList= []
# Throw an error for bad status codes
for block in response.iter_content(chunk_size=self.TOO_LONG):
output = self.bucket.upload_multipart(key,upload_id=uploadid,part_number=partNo,body=block)
logging.info("upload: %s,id:%s,part %d,status: %d" ,key,uploadid,partNo,output.status_code)
partNo +=1
partList.append(partNo)
self.bucket.complete_multipart_upload(key,upload_id=uploadid,object_parts=[{"part_number":x} for x in partList])
logging.info("uploaded: %s" ,key)
except:
logging.exception("Failed to download %s Unexpected error.",url)
return item
def is_timestamp_equal(self,url,key):
objStatus=self.bucket.head_object(key)
if objStatus.status_code == 200 :
with self.session.head(url) as response:
logging.info("key: %s source length:%s target length %s modified %s source %s",key,response.headers['Content-Length'],objStatus.headers['Content-Length'],response.headers['Last-Modified'],objStatus.headers['Last-Modified'])
return objStatus.headers['Content-Length'] == response.headers['Content-Length'] #and objStatus.headers['Last-Modified']== response.headers['Last-Modified']
return False
def open_spider(self, spider):
self.config= Config(self.qs_asscess_key,self.qs_secret_key)
self.qingstor = QingStor(self.config)
self.bucket = self.qingstor.Bucket(self.qs_bucket_name,self.qs_zone)
self.session = requests.Session()
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import os
import sh
import yaml
from behave import *
from assertpy import assert_that
from qingstor.sdk.config import Config
from qingstor.sdk.service.qingstor import QingStor
config = Config().load_user_config()
qingstor = QingStor(config)
test_config_file_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.path.pardir))
with open(test_config_file_path + '/test_config.yaml') as f:
test_data = yaml.load(f)
f.close()
bucket = qingstor.Bucket(test_data["bucket_name"] + "1", test_data["zone"])
qsctl = sh.Command("qsctl")
@when(u'create a bucket')
def step_impl(context):
qsctl("mb", "qs://{bucket}".format(bucket=test_data["bucket_name"] + "1"))
@then(u'should get a new bucket')
def step_impl(context):
context.qingstor = QingStor(config)
context.res = context.qingstor.list_buckets()
def step_impl(context):
context.qingstor = QingStor(config)
# -*- coding:utf-8 -*-
from qingstor.sdk.service.qingstor import QingStor
from qingstor.sdk.config import Config
import requests
import tempfile
# 初始化QingStor对象
config = Config('EBZAULNLJTMJJCZJWCRG',
'02MLUQZA5wZPwTrOmVWUajPhNUSFI8c5VKu2Wf0u')
qingstor = QingStor(config)
# 获取bucket列表
# output = qingstor.list_buckets()
# print(output.status_code)
# print(output['buckets'])
# 获取指定bucket对象
bucket = qingstor.Bucket('fifa-helper-bucket', 'sh1a')
def getFileByKey(key):
resp = bucket.get_object(key)
with tempfile.NamedTemporaryFile() as f:
for chunk in resp.iter_content():
f.write(chunk)
return f
def getRespByKey(key):
resp = bucket.get_object(key)
# -- bucket --
from qingstor.sdk.service.qingstor import QingStor
from qingstor.sdk.config import Config
ACCESS_KEY_ID = 'KPXLUFSRVNVNZGFCEPDT'
SECRET_ACCESS_KEY = '9RW7JW2RsIDmArXSdeHhCjYt7A9vHPs6LBT8zSEp'
prefix = 'https://mrc-lxm.pek3b.qingstor.com/'
tmp_path = '/tmp/mrc'
if sys.platform == 'win32':
tmp_path = os.path.join(os.environ['TMP'], 'mrc')
config = Config(ACCESS_KEY_ID, SECRET_ACCESS_KEY)
qingstor = QingStor(config)
bucket = qingstor.Bucket('mrc-lxm', 'pek3b')
# -- model --
import torch
from fairseq.data.data_utils import collate_tokens
from fairseq.models.roberta import RobertaModel
# -- translate --
from translate import translate
# -- utils --
def clear_unascii(s):
return ''.join([c for c in s if ascii.isascii(c)])
