def test_copy_file(self): scs = SCSBucket(self.bucket_name) try: scs.put_bucket('public-read-write') except: self.fail('create bucket:%s is failed'%self.bucket_name) scs.putFile(self.object_key, self.local_file_name) canned_acl = 'public-read' #快捷ACL metadata = {} #自定义文件属性信息 metadata['author'] = 'copied' metadata['home'] = 'beijing' metadata['age'] = '189' mimetype = 'text/plain' scs.copy(source='/'+self.bucket_name+'/'+self.object_key, key=self.object_key+'_copied', acl=canned_acl, metadata=metadata, mimetype=mimetype) copied_file_aclInfo = scs.acl_info(self.object_key+'_copied') self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in copied_file_aclInfo['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group') self.assertTrue(ACL.ACL_READ in copied_file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right') copied_file_info = scs.info(self.object_key+'_copied') #assert metadata self.assertEqual(metadata['author'], copied_file_info['metadata']['author'], 'The response metadata[\'author\'] is not match') self.assertEqual(metadata['home'], copied_file_info['metadata']['home'], 'The response metadata[\'home\'] is not match') self.assertEqual(metadata['age'], copied_file_info['metadata']['age'], 'The response metadata[\'age\'] is not match') #assert content-type self.assertEqual(mimetype, copied_file_info['headers']['content-type'], 'The response content-type is not match')
def test_get_file_meta(self): scs = SCSBucket(self.bucket_name) try: scs.put_bucket('public-read-write') except: self.fail('create bucket:%s is failed'%self.bucket_name) metadata = {} #自定义文件属性信息 metadata['author'] = 'dage' metadata['home'] = 'tianjin' metadata['age'] = '18' mimetype = 'text/plain' scs.putFile(key=self.object_key, filePath=self.local_file_name, metadata=metadata, mimetype=mimetype) metaResult = scs.meta(self.object_key) self.assertEqual(self.object_key, metaResult['File-Name'], 'The meta[\'File-Name\'] is not equals '+self.object_key) self.assertEqual(mimetype, metaResult['Type'], 'The meta[\'Type\'] is not equals '+mimetype) self.assertTrue(metaResult['Content-SHA1'] is not None, 'The meta[\'Content-SHA1\'] is None') self.assertTrue(metaResult['Content-MD5'] is not None, 'The meta[\'Content-MD5\'] is None') self.assertTrue(metaResult['Owner'] is not None, 'The meta[\'Owner\'] is None') self.assertTrue('x-amz-meta-author' in metaResult['File-Meta'], 'File-Meta dose not contains x-amz-meta-author key') self.assertEqual(metadata['author'], metaResult['File-Meta']['x-amz-meta-author'], 'The metaResult[\'File-Meta\'][\'x-amz-meta-author\'] value is not match') self.assertTrue('x-amz-meta-home' in metaResult['File-Meta'], 'File-Meta dose not contains x-amz-meta-home key') self.assertEqual(metadata['home'], metaResult['File-Meta']['x-amz-meta-home'], 'The metaResult[\'File-Meta\'][\'x-amz-meta-home\'] value is not match') self.assertTrue('x-amz-meta-age' in metaResult['File-Meta'], 'File-Meta dose not contains x-amz-meta-age key') self.assertEqual(metadata['age'], metaResult['File-Meta']['x-amz-meta-age'], 'The metaResult[\'File-Meta\'][\'x-amz-meta-age\'] value is not match')
def push(dirname): ''' 把dirname文件夹下所有文件都以相同的结构同步到sae云存储中 比如,在dirname下有如下文件: dirname/a/test.txt dirname/test.txt 推送到sae的云存储中的文件结构就是: /a/test.txt /test.txt 注意空文件夹不推送 ''' akey = os.environ['SAE_STORAGE_ACCESS_KEY'] skey = os.environ['SAE_STORAGE_SECRET_KEY'] bucketname = os.environ['SAE_BUCKET_NAME'] sinastorage.setDefaultAppInfo(akey, skey) bucket = SCSBucket(bucketname, secure=False) fpaths = [] for root, dirs, files in os.walk(dirname): ps = splitpath(root) visible = True for p in ps: if p.startswith('.'): visible = False if not visible: continue del ps[0] for f in files: local_fpath = os.path.join(root, f) ps.append(f) fpath = pathjoin(*ps) print('check :' + fpath) fpaths.append(fpath) # if not fpath.startswith(PATH_SPLITER): # fpath =PATH_SPLITER+fpath if file_exist(bucket, fpath): meta = bucket.meta(fpath) sae_sha1 = ['Content-SHA1'] fdata = open(local_fpath, 'rb').read() if len(fdata) == 0 and meta['Size'] == 0: pass else: sha1 = hashlib.sha1() sha1.update(fdata) if sae_sha1 != sha1.hexdigest(): bucket.putFile(fpath, local_fpath) else: bucket.putFile(fpath, local_fpath) sae_files = bucket.listdir(prefix='') for f in sae_files: #f=(name, isPrefix, sha1, expiration_time, modify, owner, md5, content_type, size) localf = os.path.join(dirname, f[0]) print('expecting local: ', localf) if f[0] not in fpaths and not os.path.isfile(localf): print('del ' + f[0]) del bucket[f[0]] else: print('has ' + f[0])
class Storage(object): def __init__(self, accessKey, secretKey, bucketName, bucketUrl): sinastorage.setDefaultAppInfo(accessKey, secretKey) self.s = SCSBucket(bucketName) self.bucketUrl = bucketUrl def saveImage(self, path): name = str(time.strftime('%Y%m%d/%H:%M:%S.jpg')) self.s.putFile(name, path, acl='public-read') return self.bucketUrl % name
def upload(f,bucketName,localDir,ss=0,fl=0): f = f.strip('/') s = SCSBucket(bucketName) failed = None try: result = s.putFile(f,localDir+f) print f+' upload success.' ss += 1 except Exception,e: fl += 1 failed = f
def test_put_file(self): scs = SCSBucket(self.bucket_name) try: scs.put_bucket('public-read-write') except: self.fail('create bucket:%s is failed'%self.bucket_name) canned_acl = 'public-read-write' #快捷ACL metadata = {} #自定义文件属性信息 metadata['author'] = 'dage' metadata['home'] = 'tianjin' metadata['age'] = '18' mimetype = 'text/plain' scs.putFile(key=self.object_key, filePath=self.local_file_name, acl=canned_acl , metadata=metadata, mimetype=mimetype) scsResponse = scs.get(self.object_key) CHUNK = 16 * 1024 file_content = '' from sinastorage.vendored import six while True: chunk = scsResponse.read(CHUNK) if not chunk: break if isinstance(chunk, six.text_type): file_content += chunk else: file_content += chunk.decode("utf-8") #assert file_content # self.assertEqual(content, file_content, 'The uploaded file content is not metch local.%s'%file_content) #assert metadata self.assertEqual(metadata['author'], scsResponse.responseHeaders['x-amz-meta-author'], 'The response metadata[\'author\'] is not match') self.assertEqual(metadata['home'], scsResponse.responseHeaders['x-amz-meta-home'], 'The response metadata[\'home\'] is not match') self.assertEqual(metadata['age'], scsResponse.responseHeaders['x-amz-meta-age'], 'The response metadata[\'age\'] is not match') #assert content-type self.assertEqual(mimetype, scsResponse.responseHeaders['content-type'], 'The response content-type is not match') #assert acl aclResult = scs.acl_info(self.object_key) self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in aclResult['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group') self.assertTrue(ACL.ACL_READ in aclResult['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right') self.assertTrue(ACL.ACL_WRITE in aclResult['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right')
def test_get_file(self): scs = SCSBucket(self.bucket_name) try: scs.put_bucket('public-read-write') except: self.fail('create bucket:%s is failed'%self.bucket_name) scs.putFile(self.object_key, self.local_file_name) scsResponse = scs.get(self.object_key) CHUNK = 16 * 1024 with open(self.local_file_name+'.downloaded', 'wb') as fp: while True: chunk = scsResponse.read(CHUNK) if not chunk: break fp.write(bytes(chunk) ) import os self.assertTrue(os.path.exists(self.local_file_name+'.downloaded'), 'The download file dose not exists') self.assertEqual(str(os.stat(self.local_file_name).st_size), scsResponse.responseHeaders['content-length'], 'The download file size dose not match local file')
def create(self, com): if com.path[-1] != '/': self.new_item_start.emit(os.path.basename(com.path)) cloud_path = "haoc/nodes/%s" % com.path bucket = SCSBucket(BUCKET_NAME) local_path = "%s/data/%s/nodes/%s" % ( HaocUtils.get_root_path(), HaocUtils.Config.get_ak(), com.path) if cloud_path[-1] == '/': bucket.put(cloud_path, '') else: shall_upload = True if not os.path.exists("%s.nod" % local_path): print "File not found:%s" % local_path return lt = int(os.path.getmtime("%s.nod" % local_path)) if self.from_agenda: try: bucket["%s.nod" % cloud_path] except sinastorage.bucket.KeyNotFound: pass else: info = bucket.info("%s.nod" % cloud_path) ct = int(info.get('metadata').get('lmt')) if lt < ct: shall_upload = False if shall_upload: bucket.putFile("%s.nod" % cloud_path, "%s.nod" % local_path, self.call_back) bucket.update_meta("%s.nod" % cloud_path, {'lmt': str(lt)}) bucket.update_meta("%s.nod" % cloud_path, {'hver': hou.applicationVersionString()}) if os.path.exists("%s.hlp" % local_path): bucket.putFile("%s.hlp" % cloud_path, "%s.hlp" % local_path) else: open("%s.nod" % local_path, 'w').close()
class Uploader(): def __init__(self, host, accesskey, secretkey, bucket_name, remote_dir): self.host = host self.bucket_name = bucket_name self.remote_dir = remote_dir sinastorage.setDefaultAppInfo(accesskey, secretkey) self.handler = SCSBucket(self.bucket_name) def do(self, url, filepath_list, is_file=True): remote_list = {} sub_dir = self.get_subdir(url) if filepath_list: for _url, filepath, filename in filepath_list: if not filepath: continue filename = filename if filename else str(time.time()) + ".html" print("#INFO: uploader ", filename, sub_dir) sys.stdout.flush() try: remote_link = self.upload_file(filepath, filename, sub_dir, is_file) remote_list[_url] = remote_link except Exception as e: print("#ERR: upload filed failed, %s, %s", filename, str(e)) continue return remote_list @my_utils.time_limit(5) def upload_file(self, local_path, local_name, sub_dir="", is_file=True): if not local_path or not local_name: return "" if sub_dir: sub_dir = sub_dir + "/" remote_path = self.remote_dir + "/" + sub_dir + local_name ret = self._upload_file(remote_path, local_path, is_file) self.set_acl(remote_path) return ret def _upload_file(self, remote_path, local_path, is_file=True): if is_file: ret = self.handler.putFile(remote_path, local_path) else: ret = self.handler.put(remote_path, local_path) return self.host + "/" + self.bucket_name + "/" + remote_path if ret else "" def set_acl(self, remote_path): acl = {} acl[ACL.ACL_GROUP_ANONYMOUSE] = [ACL.ACL_READ] acl[ACL.ACL_GROUP_CANONICAL] = [ACL.ACL_READ_ACP, ACL.ACL_WRITE_ACP] self.handler.update_acl(remote_path, acl) def get_subdir(self, url): url_p = urlparse.urlparse(url) return url_p.hostname.replace(".", "_")
def fun(_time_out): cloud_path = "haoc/nodes/%s.hlp" % self.path bucket = SCSBucket(BUCKET_NAME, timeout=_time_out) bucket.putFile(cloud_path, local_path)
from sinastorage.bucket import SCSBucket import sinastorage import sys sys.path.append("../") import config import time uploadedAmount = 0.0 def putFileCallback(size, progress): print(size, progress) sinastorage.setDefaultAppInfo(config.SinaSCSAccessKey, config.SinaSCSSecretKey) s = SCSBucket(config.SinaSCSBucketName) # s.putFile('1.jpg', 'selfie.jpg', putFileCallback) ret = s.putFile(str(time.strftime('%Y%m%d/%H:%M:%S')) + '.jpg', 'selfie.jpg', acl='public-read') print('complete!') print(ret.info())
#coding:utf-8 ''' this file is meant to invoking scs-sdk to make connection with the cloud storage ''' import sinastorage from sinastorage.bucket import SCSBucket sinastorage.setDefaultAppInfo('XXXXXXXXXXXXXXXXXXXXXXXX','xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') s=SCSBucket('mystorage',secure=True) #print s #s.putFile('mystorage','D:\\a.jpg') test s.putFile('math1','D:\\Mathematical Modeling\\math1.rar')
# -*- coding:utf-8 -*- __author__ = 'walkskyer' """ """ from sinastorage.bucket import SCSBucket import sinastorage if __name__ == "__main__": accesskey = '1ejzfhbkgyCTyFbzfFiL' sccretkey = 'SINA0000000001EJZFHB' #设置access_key,secret_key sinastorage.setDefaultAppInfo(accesskey, sccretkey) s = SCSBucket('wk-test',secure=True) # secure=True 采用https访问方式 print s #文件内容 # scsResponse = s.put('test.txt',u'hello world!') # print scsResponse #文件路径 localfile = 'assets/0004.jpg' scsResponse1 = s.putFile('test.jpg', localfile) print scsResponse1
def put_file(): s = SCSBucket('create-a-bucket11') # print s.put('sss.txt',u'测试测试testtest') print s.putFile('111111qt-opensource-mac-4.8.6.dmg', '/Users/hanchao/Desktop/qt-opensource-mac-4.8.6.dmg', putFileCallback)
# -*- coding: utf-8 -*- from sinastorage.bucket import SCSBucket import sinastorage from zipfile import ZipFile from io import BytesIO with ZipFile('test.zip', 'r') as zf: sinastorage.setDefaultAppInfo('16wn9n74cnZJFjffXU3K', '50706013e9a806c252aafb03ad2a3b51174e36ad') s = SCSBucket('saepcs', secure=False) # secure=True 采用https访问方式 scsResponse = s.putFile('stock/test.zip', 'test.zip')