def test_copy_file(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
scs.putFile(self.object_key, self.local_file_name)
canned_acl = 'public-read' #快捷ACL
metadata = {} #自定义文件属性信息
metadata['author'] = 'copied'
metadata['home'] = 'beijing'
metadata['age'] = '189'
mimetype = 'text/plain'
scs.copy(source='/'+self.bucket_name+'/'+self.object_key,
key=self.object_key+'_copied', acl=canned_acl,
metadata=metadata, mimetype=mimetype)
copied_file_aclInfo = scs.acl_info(self.object_key+'_copied')
self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in copied_file_aclInfo['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group')
self.assertTrue(ACL.ACL_READ in copied_file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right')
copied_file_info = scs.info(self.object_key+'_copied')
#assert metadata
self.assertEqual(metadata['author'], copied_file_info['metadata']['author'], 'The response metadata[\'author\'] is not match')
self.assertEqual(metadata['home'], copied_file_info['metadata']['home'], 'The response metadata[\'home\'] is not match')
self.assertEqual(metadata['age'], copied_file_info['metadata']['age'], 'The response metadata[\'age\'] is not match')
#assert content-type
self.assertEqual(mimetype, copied_file_info['headers']['content-type'], 'The response content-type is not match')
def test_get_file_meta(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
metadata = {} #自定义文件属性信息
metadata['author'] = 'dage'
metadata['home'] = 'tianjin'
metadata['age'] = '18'
mimetype = 'text/plain'
scs.putFile(key=self.object_key, filePath=self.local_file_name,
metadata=metadata, mimetype=mimetype)
metaResult = scs.meta(self.object_key)
self.assertEqual(self.object_key, metaResult['File-Name'], 'The meta[\'File-Name\'] is not equals '+self.object_key)
self.assertEqual(mimetype, metaResult['Type'], 'The meta[\'Type\'] is not equals '+mimetype)
self.assertTrue(metaResult['Content-SHA1'] is not None, 'The meta[\'Content-SHA1\'] is None')
self.assertTrue(metaResult['Content-MD5'] is not None, 'The meta[\'Content-MD5\'] is None')
self.assertTrue(metaResult['Owner'] is not None, 'The meta[\'Owner\'] is None')
self.assertTrue('x-amz-meta-author' in metaResult['File-Meta'], 'File-Meta dose not contains x-amz-meta-author key')
self.assertEqual(metadata['author'], metaResult['File-Meta']['x-amz-meta-author'], 'The metaResult[\'File-Meta\'][\'x-amz-meta-author\'] value is not match')
self.assertTrue('x-amz-meta-home' in metaResult['File-Meta'], 'File-Meta dose not contains x-amz-meta-home key')
self.assertEqual(metadata['home'], metaResult['File-Meta']['x-amz-meta-home'], 'The metaResult[\'File-Meta\'][\'x-amz-meta-home\'] value is not match')
self.assertTrue('x-amz-meta-age' in metaResult['File-Meta'], 'File-Meta dose not contains x-amz-meta-age key')
self.assertEqual(metadata['age'], metaResult['File-Meta']['x-amz-meta-age'], 'The metaResult[\'File-Meta\'][\'x-amz-meta-age\'] value is not match')
def push(dirname):
'''
把dirname文件夹下所有文件都以相同的结构同步到sae云存储中
比如，在dirname下有如下文件：
dirname/a/test.txt
dirname/test.txt
推送到sae的云存储中的文件结构就是：
/a/test.txt
/test.txt
注意空文件夹不推送
'''
akey = os.environ['SAE_STORAGE_ACCESS_KEY']
skey = os.environ['SAE_STORAGE_SECRET_KEY']
bucketname = os.environ['SAE_BUCKET_NAME']
sinastorage.setDefaultAppInfo(akey, skey)
bucket = SCSBucket(bucketname, secure=False)
fpaths = []
for root, dirs, files in os.walk(dirname):
ps = splitpath(root)
visible = True
for p in ps:
if p.startswith('.'):
visible = False
if not visible:
continue
del ps[0]
for f in files:
local_fpath = os.path.join(root, f)
ps.append(f)
fpath = pathjoin(*ps)
print('check :' + fpath)
fpaths.append(fpath)
# if not fpath.startswith(PATH_SPLITER):
# fpath =PATH_SPLITER+fpath
if file_exist(bucket, fpath):
meta = bucket.meta(fpath)
sae_sha1 = ['Content-SHA1']
fdata = open(local_fpath, 'rb').read()
if len(fdata) == 0 and meta['Size'] == 0:
pass
else:
sha1 = hashlib.sha1()
sha1.update(fdata)
if sae_sha1 != sha1.hexdigest():
bucket.putFile(fpath, local_fpath)
else:
bucket.putFile(fpath, local_fpath)
sae_files = bucket.listdir(prefix='')
for f in sae_files:
#f=(name, isPrefix, sha1, expiration_time, modify, owner, md5, content_type, size)
localf = os.path.join(dirname, f[0])
print('expecting local: ', localf)
if f[0] not in fpaths and not os.path.isfile(localf):
print('del ' + f[0])
del bucket[f[0]]
else:
print('has ' + f[0])
class Storage(object):
def __init__(self, accessKey, secretKey, bucketName, bucketUrl):
sinastorage.setDefaultAppInfo(accessKey, secretKey)
self.s = SCSBucket(bucketName)
self.bucketUrl = bucketUrl
def saveImage(self, path):
name = str(time.strftime('%Y%m%d/%H:%M:%S.jpg'))
self.s.putFile(name, path, acl='public-read')
return self.bucketUrl % name
def upload(f,bucketName,localDir,ss=0,fl=0):
f = f.strip('/')
s = SCSBucket(bucketName)
failed = None
try:
result = s.putFile(f,localDir+f)
print f+' upload success.'
ss += 1
except Exception,e:
fl += 1
failed = f
def test_put_file(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
canned_acl = 'public-read-write' #快捷ACL
metadata = {} #自定义文件属性信息
metadata['author'] = 'dage'
metadata['home'] = 'tianjin'
metadata['age'] = '18'
mimetype = 'text/plain'
scs.putFile(key=self.object_key, filePath=self.local_file_name, acl=canned_acl
, metadata=metadata, mimetype=mimetype)
scsResponse = scs.get(self.object_key)
CHUNK = 16 * 1024
file_content = ''
from sinastorage.vendored import six
while True:
chunk = scsResponse.read(CHUNK)
if not chunk: break
if isinstance(chunk, six.text_type):
file_content += chunk
else:
file_content += chunk.decode("utf-8")
#assert file_content
# self.assertEqual(content, file_content, 'The uploaded file content is not metch local.%s'%file_content)
#assert metadata
self.assertEqual(metadata['author'], scsResponse.responseHeaders['x-amz-meta-author'], 'The response metadata[\'author\'] is not match')
self.assertEqual(metadata['home'], scsResponse.responseHeaders['x-amz-meta-home'], 'The response metadata[\'home\'] is not match')
self.assertEqual(metadata['age'], scsResponse.responseHeaders['x-amz-meta-age'], 'The response metadata[\'age\'] is not match')
#assert content-type
self.assertEqual(mimetype, scsResponse.responseHeaders['content-type'], 'The response content-type is not match')
#assert acl
aclResult = scs.acl_info(self.object_key)
self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in aclResult['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group')
self.assertTrue(ACL.ACL_READ in aclResult['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right')
self.assertTrue(ACL.ACL_WRITE in aclResult['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right')
def test_get_file(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
scs.putFile(self.object_key, self.local_file_name)
scsResponse = scs.get(self.object_key)
CHUNK = 16 * 1024
with open(self.local_file_name+'.downloaded', 'wb') as fp:
while True:
chunk = scsResponse.read(CHUNK)
if not chunk: break
fp.write(bytes(chunk) )
import os
self.assertTrue(os.path.exists(self.local_file_name+'.downloaded'), 'The download file dose not exists')
self.assertEqual(str(os.stat(self.local_file_name).st_size),
scsResponse.responseHeaders['content-length'], 'The download file size dose not match local file')
def create(self, com):
if com.path[-1] != '/':
self.new_item_start.emit(os.path.basename(com.path))
cloud_path = "haoc/nodes/%s" % com.path
bucket = SCSBucket(BUCKET_NAME)
local_path = "%s/data/%s/nodes/%s" % (
HaocUtils.get_root_path(), HaocUtils.Config.get_ak(), com.path)
if cloud_path[-1] == '/':
bucket.put(cloud_path, '')
else:
shall_upload = True
if not os.path.exists("%s.nod" % local_path):
print "File not found:%s" % local_path
return
lt = int(os.path.getmtime("%s.nod" % local_path))
if self.from_agenda:
try:
bucket["%s.nod" % cloud_path]
except sinastorage.bucket.KeyNotFound:
pass
else:
info = bucket.info("%s.nod" % cloud_path)
ct = int(info.get('metadata').get('lmt'))
if lt < ct:
shall_upload = False
if shall_upload:
bucket.putFile("%s.nod" % cloud_path, "%s.nod" % local_path,
self.call_back)
bucket.update_meta("%s.nod" % cloud_path, {'lmt': str(lt)})
bucket.update_meta("%s.nod" % cloud_path,
{'hver': hou.applicationVersionString()})
if os.path.exists("%s.hlp" % local_path):
bucket.putFile("%s.hlp" % cloud_path,
"%s.hlp" % local_path)
else:
open("%s.nod" % local_path, 'w').close()
class Uploader():
def __init__(self, host, accesskey, secretkey, bucket_name, remote_dir):
self.host = host
self.bucket_name = bucket_name
self.remote_dir = remote_dir
sinastorage.setDefaultAppInfo(accesskey, secretkey)
self.handler = SCSBucket(self.bucket_name)
def do(self, url, filepath_list, is_file=True):
remote_list = {}
sub_dir = self.get_subdir(url)
if filepath_list:
for _url, filepath, filename in filepath_list:
if not filepath:
continue
filename = filename if filename else str(time.time()) + ".html"
print("#INFO: uploader ", filename, sub_dir)
sys.stdout.flush()
try:
remote_link = self.upload_file(filepath, filename, sub_dir,
is_file)
remote_list[_url] = remote_link
except Exception as e:
print("#ERR: upload filed failed, %s, %s", filename,
str(e))
continue
return remote_list
@my_utils.time_limit(5)
def upload_file(self, local_path, local_name, sub_dir="", is_file=True):
if not local_path or not local_name:
return ""
if sub_dir:
sub_dir = sub_dir + "/"
remote_path = self.remote_dir + "/" + sub_dir + local_name
ret = self._upload_file(remote_path, local_path, is_file)
self.set_acl(remote_path)
return ret
def _upload_file(self, remote_path, local_path, is_file=True):
if is_file:
ret = self.handler.putFile(remote_path, local_path)
else:
ret = self.handler.put(remote_path, local_path)
return self.host + "/" + self.bucket_name + "/" + remote_path if ret else ""
def set_acl(self, remote_path):
acl = {}
acl[ACL.ACL_GROUP_ANONYMOUSE] = [ACL.ACL_READ]
acl[ACL.ACL_GROUP_CANONICAL] = [ACL.ACL_READ_ACP, ACL.ACL_WRITE_ACP]
self.handler.update_acl(remote_path, acl)
def get_subdir(self, url):
url_p = urlparse.urlparse(url)
return url_p.hostname.replace(".", "_")
def fun(_time_out):
cloud_path = "haoc/nodes/%s.hlp" % self.path
bucket = SCSBucket(BUCKET_NAME, timeout=_time_out)
bucket.putFile(cloud_path, local_path)
from sinastorage.bucket import SCSBucket
import sinastorage
import sys
sys.path.append("../")
import config
import time
uploadedAmount = 0.0
def putFileCallback(size, progress):
print(size, progress)
sinastorage.setDefaultAppInfo(config.SinaSCSAccessKey, config.SinaSCSSecretKey)
s = SCSBucket(config.SinaSCSBucketName)
# s.putFile('1.jpg', 'selfie.jpg', putFileCallback)
ret = s.putFile(str(time.strftime('%Y%m%d/%H:%M:%S')) + '.jpg',
'selfie.jpg',
acl='public-read')
print('complete!')
print(ret.info())
#coding:utf-8
'''
this file is meant to invoking scs-sdk
to make connection with the cloud storage
'''
import sinastorage
from sinastorage.bucket import SCSBucket
sinastorage.setDefaultAppInfo('XXXXXXXXXXXXXXXXXXXXXXXX','xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx')
s=SCSBucket('mystorage',secure=True)
#print s
#s.putFile('mystorage','D:\\a.jpg') test
s.putFile('math1','D:\\Mathematical Modeling\\math1.rar')
# -*- coding:utf-8 -*-
__author__ = 'walkskyer'
"""
"""
from sinastorage.bucket import SCSBucket
import sinastorage
if __name__ == "__main__":
accesskey = '1ejzfhbkgyCTyFbzfFiL'
sccretkey = 'SINA0000000001EJZFHB'
#设置access_key,secret_key
sinastorage.setDefaultAppInfo(accesskey, sccretkey)
s = SCSBucket('wk-test',secure=True) # secure=True 采用https访问方式
print s
#文件内容
# scsResponse = s.put('test.txt',u'hello world!')
# print scsResponse
#文件路径
localfile = 'assets/0004.jpg'
scsResponse1 = s.putFile('test.jpg', localfile)
print scsResponse1
def put_file():
s = SCSBucket('create-a-bucket11')
# print s.put('sss.txt',u'测试测试testtest')
print s.putFile('111111qt-opensource-mac-4.8.6.dmg',
'/Users/hanchao/Desktop/qt-opensource-mac-4.8.6.dmg',
putFileCallback)
# -*- coding: utf-8 -*-
from sinastorage.bucket import SCSBucket
import sinastorage
from zipfile import ZipFile
from io import BytesIO
with ZipFile('test.zip', 'r') as zf:
sinastorage.setDefaultAppInfo('16wn9n74cnZJFjffXU3K', '50706013e9a806c252aafb03ad2a3b51174e36ad')
s = SCSBucket('saepcs', secure=False) # secure=True 采用https访问方式
scsResponse = s.putFile('stock/test.zip', 'test.zip')
