def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
if self.key.rfind('/') == len(self.key)-1 :
m = (("prefix", smart_str(self.key)),
("delimiter", '/'),
("max-keys", 5),
("formatter","json"))
args = dict((str(k), str(v)) for (k, v) in m if v is not None)
response = s.send(s.request(key='', args=args))
files_generator = SCSListing.parse(response)
if files_generator.contents_quantity > 0 or files_generator.common_prefixes_quantity > 0 :
for item in files_generator:
if cmp(item[0],self.key) != 0:
self.emitter.emit(QtCore.SIGNAL("DeleteObjectForbidden(PyQt_PyObject,PyQt_PyObject)"), self, u'不能删除非空目录(前缀) ')
return
# s = SCSBucket(self.bucketName)
self.response = s.send(s.request(method="DELETE", key=self.key))
except SCSError, e:
self.state = RunnableState.DID_FAILED
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.emitter.emit(QtCore.SIGNAL("DeleteObjectDidFailed(PyQt_PyObject,PyQt_PyObject)"), self, e.msg)
return
def upload(self):
''' 普通上传 '''
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
with FileWithCallback(self.filePath, 'rb',
self.uploadCallBack) as fileWithCallback:
scsResponse = s.putFileByHeaders(
'%s%s' % (self.prefix, os.path.basename(self.filePath)),
fileWithCallback)
self.response = scsResponse
except SCSError, e:
print e
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
if isinstance(e, ManualCancel): #手动取消
self.state = RunnableState.DID_CANCELED
self.response._responseBody = u'手动取消'
self.emitter.emit(
QtCore.SIGNAL(
"fileUploadDidCanceled(PyQt_PyObject,PyQt_PyObject)"),
self, e.msg)
else:
self.state = RunnableState.DID_FAILED
self.response._responseBody = e.data
self.emitter.emit(
QtCore.SIGNAL(
"fileUploadDidFailed(PyQt_PyObject,PyQt_PyObject)"),
self, e.msg)
return
def update_file_acl():
s = SCSBucket('test11')
acl = {}
acl[ACL.ACL_GROUP_ANONYMOUSE] = [ACL.ACL_READ]
acl[ACL.ACL_GROUP_CANONICAL] = [ACL.ACL_READ_ACP,ACL.ACL_WRITE_ACP]
s.update_acl('sss.txt', acl)
def blog():
sinastorage.setDefaultAppInfo(
'***********************', '*************************************')
s = SCSBucket('sundae')
url1 = s.make_url_authed('picture/beijing.jpg')
url2 = s.make_url_authed('picture/ziti.png')
return render_template('blog.html', url0=url1, url1=url2)
def fun(_time_out):
if self.lis is not None:
self.lis = []
bucket = SCSBucket(BUCKET_NAME, timeout=_time_out)
items_generator = bucket.listdir(prefix='haoc/nodes/', limit=10000)
access_key = HaocUtils.Config.get_ak()
for item in items_generator:
common_name = item[0][11:]
local_name = "%s/data/%s/nodes/%s" % (
HaocUtils.get_root_path(), access_key, common_name)
if self.lis is not None:
if common_name[-1] == '/' or common_name[-4:] == '.nod':
self.lis.append(common_name)
if common_name[-1] == '/':
if not os.path.exists(local_name):
os.makedirs(local_name)
elif common_name[-4:]:
if os.path.exists(local_name):
lt = int(os.path.getmtime("%s.nod" % local_name[:-4]))
ct = int(
bucket.info(
"%s.nod" %
item[0][:-4]).get('metadata').get('lmt'))
# Compare local file modified time with cloud file modified time
if lt < ct:
open(local_name, "w").close()
if is_manual_sync:
print "Updated: %s" % common_name
else:
if not os.path.exists(os.path.dirname(local_name)):
os.makedirs(os.path.dirname(local_name))
open(local_name, "w").close()
if is_manual_sync:
print "Created: %s" % common_name
def html_form():
sinastorage.setDefaultAppInfo(
'***********************', '*************************************')
s = SCSBucket('eggtart')
url = s.make_url_authed('picture/bj1.gif')
whuurl = s.make_url_authed('picture/whu.jpg')
return render_template('html_form.html', url=url, url1=whuurl)
def test_put_file_relax(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
scs.put(self.object_key, 'fileContent...fileContent...fileContent...fileContent...fileContent...')
orign_file_metaResult = scs.meta(self.object_key)
canned_acl = 'public-read-write' #快捷ACL
metadata = {} #自定义文件属性信息
metadata['author'] = 'dage'
metadata['home'] = 'tianjin'
metadata['age'] = '18'
mimetype = 'text/plain'
scs.put_relax(key=self.object_key+'_relax', sina_sha1=orign_file_metaResult['Content-SHA1'],
s_sina_length=orign_file_metaResult['Size'], acl=canned_acl,
metadata=metadata, mimetype=mimetype)
relax_file_aclInfo = scs.acl_info(self.object_key+'_relax')
self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in relax_file_aclInfo['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group')
self.assertTrue(ACL.ACL_READ in relax_file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right')
self.assertTrue(ACL.ACL_WRITE in relax_file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right')
relax_file_info = scs.info(self.object_key+'_relax')
#assert metadata
self.assertEqual(metadata['author'], relax_file_info['metadata']['author'], 'The response metadata[\'author\'] is not match')
self.assertEqual(metadata['home'], relax_file_info['metadata']['home'], 'The response metadata[\'home\'] is not match')
self.assertEqual(metadata['age'], relax_file_info['metadata']['age'], 'The response metadata[\'age\'] is not match')
#assert content-type
self.assertEqual(mimetype, relax_file_info['headers']['content-type'], 'The response content-type is not match')
def __init__(self, host, accesskey, secretkey, bucket_name, remote_dir):
self.host = host
self.bucket_name = bucket_name
self.remote_dir = remote_dir
sinastorage.setDefaultAppInfo(accesskey, secretkey)
self.handler = SCSBucket(self.bucket_name)
def copy_file():
s = SCSBucket('asdasdasdasd')
"""
注意：
source 必须从bucket开始，如：'/cloud0/aaa.txt'
"""
s.copy('/asdasdasdasd/aaaa.txt', 'aaaa22111.txt')
def who():
sinastorage.setDefaultAppInfo(
'***********************', '*************************************')
s = SCSBucket('sundae')
url1 = s.make_url_authed('picture/who.png')
url0 = s.make_url_authed('picture/bj1.gif')
url2 = s.make_url_authed('picture/i.png')
return render_template('who.html', url0=url0, url1=url1, url2=url2)
def listFile(bucketName): Files = [] s = SCSBucket(bucketName) files = s.listdir() for i in files: #i = (name, isPrefix, sha1, expiration_time, modify, owner, md5, content_type, size) Files.append(i) return Files
def test_list_bucket(self):
scs = SCSBucket()
bucket_gen = scs.list_buckets()
self.assertTrue(bucket_gen is not None, 'List bucket result is None')
for bucket in bucket_gen:
bucketStr = str(bucket)
self.assertTrue(bucket[0] is not None, 'bucket:%s name is None'%bucketStr)
self.assertTrue(bucket[1] is not None, 'bucket:%s CreationDate is None'%bucketStr)
def test_remove_bucket(self):
scs = SCSBucket(self.bucket_name)
scs.put_bucket()
try:
scs.delete_bucket()
except:
self.fail('Remove bucket:%s is failed'%self.bucket_name)
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
# self.response = s[self.key]
headers = {}
''' 若文件存在，则received等于已下载的文件大小 '''
if os.path.exists(self.tmpFilePath):
self.received = os.stat(self.tmpFilePath).st_size
headers['If-Range'] = u'"%s"'%self.fileMD5
headers['Range'] = u'bytes=%d-'%self.received
self.response = s.send(s.request(key=self.key,headers=headers))
statusCode = getattr(self.response.urllib2Response, "code", None)
responseHeaders = dict(self.response.urllib2Response.info())
if statusCode == 200:
if "content-length" in responseHeaders:
self.total = long(responseHeaders["content-length"])
else:
raise ValueError("Content-Length not returned!!")
elif statusCode == 206:
''' 用于断点续传时获取文件总大小 '''
if "content-range" in responseHeaders:
content_range = responseHeaders["content-range"]
self.total = long(content_range[content_range.rfind('/')+1:])
else:
raise ValueError("Content-Length not returned!!")
lastTimestamp = time.time()
CHUNK = 16 * 1024
_received_tmp = 0 #内部临时变量
with open(self.tmpFilePath, 'ab') as fp:
while True:
if self.isAbort:
self.state = RunnableState.DID_CANCELED
self.response._responseBody = u'手动取消'
self.emitter.emit(QtCore.SIGNAL("DownloadObjectDidCanceled(PyQt_PyObject)"), self)
return
chunk = self.response.read(CHUNK)
if not chunk: break
fp.write(chunk)
_received_tmp += len(chunk)
if time.time() - lastTimestamp >= 1.0:
self.downloadCallBack(_received_tmp)
lastTimestamp = time.time()
_received_tmp = 0
except SCSError, e:
self.state = RunnableState.DID_FAILED
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.emitter.emit(QtCore.SIGNAL("DownloadObjectDidFailed(PyQt_PyObject,PyQt_PyObject)"), self, e.msg)
return
def listBucket(): Buckets = [] s = SCSBucket() buckets_generator = s.list_buckets() for bucket in buckets_generator: bucketName = bucket[0] bMeta = SCSBucket(bucketName).meta() Buckets.append(bMeta) return Buckets
def rename(self, com, time_out):
old_cloud_name = "haoc/nodes/%s" % com.old_name
new_cloud_name = "haoc/nodes/%s" % com.new_name
bucket = SCSBucket(BUCKET_NAME, timeout=time_out)
try:
if new_cloud_name[-1] == '/':
bucket.copy(source="/%s/%s" % (BUCKET_NAME, old_cloud_name),
key=new_cloud_name)
del bucket[old_cloud_name]
else:
shall_do = True
if self.from_agenda:
try:
bucket["%s.nod" % new_cloud_name]
except sinastorage.bucket.KeyNotFound:
pass
else:
info = bucket.info("%s.nod" % new_cloud_name)
ct = float(info.get('metadata').get('lmt'))
if com.op_time < ct:
shall_do = False
if shall_do:
bucket.copy(source="/%s/%s.nod" %
(BUCKET_NAME, old_cloud_name),
key="%s.nod" % new_cloud_name)
del bucket["%s.nod" % old_cloud_name]
bucket.copy(source="/%s/%s.hlp" %
(BUCKET_NAME, old_cloud_name),
key="%s.hlp" % new_cloud_name)
del bucket["%s.hlp" % old_cloud_name]
except sinastorage.bucket.KeyNotFound:
pass
def test_copy_file(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
scs.putFile(self.object_key, self.local_file_name)
canned_acl = 'public-read' #快捷ACL
metadata = {} #自定义文件属性信息
metadata['author'] = 'copied'
metadata['home'] = 'beijing'
metadata['age'] = '189'
mimetype = 'text/plain'
scs.copy(source='/'+self.bucket_name+'/'+self.object_key,
key=self.object_key+'_copied', acl=canned_acl,
metadata=metadata, mimetype=mimetype)
copied_file_aclInfo = scs.acl_info(self.object_key+'_copied')
self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in copied_file_aclInfo['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group')
self.assertTrue(ACL.ACL_READ in copied_file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right')
copied_file_info = scs.info(self.object_key+'_copied')
#assert metadata
self.assertEqual(metadata['author'], copied_file_info['metadata']['author'], 'The response metadata[\'author\'] is not match')
self.assertEqual(metadata['home'], copied_file_info['metadata']['home'], 'The response metadata[\'home\'] is not match')
self.assertEqual(metadata['age'], copied_file_info['metadata']['age'], 'The response metadata[\'age\'] is not match')
#assert content-type
self.assertEqual(mimetype, copied_file_info['headers']['content-type'], 'The response content-type is not match')
def test_update_file_meta(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
content = u'this is a file content text!!'
scs.put(self.object_key, content)
canned_acl = 'public-read-write' #快捷ACL
metadata = {} #自定义文件属性信息
metadata['author'] = 'dage'
metadata['home'] = 'tianjin'
metadata['age'] = '18'
mimetype = 'text/plain'
scs.update_meta(self.object_key, metadata=metadata, acl=canned_acl, mimetype=mimetype)
file_info = scs.info(self.object_key)
#assert metadata
self.assertEqual(metadata['author'], file_info['metadata']['author'], 'The response metadata[\'author\'] is not match')
self.assertEqual(metadata['home'], file_info['metadata']['home'], 'The response metadata[\'home\'] is not match')
self.assertEqual(metadata['age'], file_info['metadata']['age'], 'The response metadata[\'age\'] is not match')
#assert content-type
# print file_info['headers']
#TODO:服务器问题 self.assertEqual(mimetype, file_info['headers']['content-type'], 'The response content-type is not match')
#acl
file_aclInfo = scs.acl_info(self.object_key)
self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in file_aclInfo['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group')
self.assertTrue(ACL.ACL_READ in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right')
self.assertTrue(ACL.ACL_WRITE in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right')
class Storage(object):
def __init__(self, accessKey, secretKey, bucketName, bucketUrl):
sinastorage.setDefaultAppInfo(accessKey, secretKey)
self.s = SCSBucket(bucketName)
self.bucketUrl = bucketUrl
def saveImage(self, path):
name = str(time.strftime('%Y%m%d/%H:%M:%S.jpg'))
self.s.putFile(name, path, acl='public-read')
return self.bucketUrl % name
def test_create_bucket_with_acl(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
metaResult = scs.meta()
self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in metaResult['ACL'], 'The bucket:%s acl dose not contains GRPS000000ANONYMOUSE group'%self.bucket_name)
self.assertTrue(ACL.ACL_READ in metaResult['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The bucket:%s acl GRPS000000ANONYMOUSE group hasn\'t read right'%self.bucket_name)
self.assertTrue(ACL.ACL_WRITE in metaResult['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The bucket:%s acl GRPS000000ANONYMOUSE group hasn\'t write right'%self.bucket_name)
def upload(f,bucketName,localDir,ss=0,fl=0):
f = f.strip('/')
s = SCSBucket(bucketName)
failed = None
try:
result = s.putFile(f,localDir+f)
print f+' upload success.'
ss += 1
except Exception,e:
fl += 1
failed = f
def run(self):
s = SCSBucket(self.bucketName)
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
self.response = s.send(s.request(method="DELETE", key=None))
except SCSError, e:
self.state = RunnableState.DID_FAILED
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.emitter.emit(QtCore.SIGNAL("DeleteBucketRunnableDidFailed(PyQt_PyObject,PyQt_PyObject)"), self, e.msg)
return
def obj_upload(data, fileTag):
""" Upload photo by object. """
sinastorage.setDefaultAppInfo(API_KEY, API_SECRET)
s = SCSBucket('sharephotos')
path = time.ctime().replace(' ', '_')
# if '/' in file name there will be problems
filename = fileTag[:6] + '_' + path + '.jpg' # use jpg temporary
scs_response = s.put(filename, data)
s.update_acl(filename, acl)
url = s.make_url(filename) # get url of image in the storage
return url
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
scsResponse = s.update_acl(self.key, self.acl)
self.response = scsResponse
except SCSError, e:
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.state = RunnableState.DID_FAILED
self.emitter.emit(QtCore.SIGNAL("UpdateFileACLDidFailed(PyQt_PyObject,PyQt_PyObject)"), self, e.msg)
return
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
self.response = s.put_bucket()
self.response.close()
except SCSError, e:
self.state = RunnableState.DID_FAILED
self.emitter.emit(QtCore.SIGNAL("CreateBucketDidFailed(PyQt_PyObject,PyQt_PyObject)"), self, e.msg)
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
return
def url_upload(imgSrc='http://www.w3school.com.cn/i/site_photoref.jpg'):
""" Upload photo by url. """
sinastorage.setDefaultAppInfo(API_KEY, API_SECRET)
s = SCSBucket('sharephotos')
data = urllib2.urlopen(imgSrc).read()
path = imgSrc.split('/')[2] + '/%s__' % time.ctime()
# if '/' in file name there will be problems
filename = path + imgSrc.replace('/', '_')
scs_response = s.put(filename, data)
s.update_acl(filename, acl)
url = s.make_url(filename) # get url of image in the storage
return url
def test_update_file_acl(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
content = u'this is a file content text!!'
canned_acl = 'private' #快捷ACL
scs.put(key=self.object_key, data=content, acl=canned_acl)
#设置ACL
acl = {}
acl[ACL.ACL_GROUP_ANONYMOUSE] = [ACL.ACL_READ, ACL.ACL_WRITE, ACL.ACL_READ_ACP]
acl[ACL.ACL_GROUP_CANONICAL] = [ACL.ACL_READ_ACP,ACL.ACL_WRITE_ACP, ACL.ACL_READ]
user_id = 'SINA0000001001AABBCC'
acl[user_id] = [ACL.ACL_WRITE, ACL.ACL_READ]
scs.update_acl(self.object_key, acl)
#assert
file_aclInfo = scs.acl_info(self.object_key)
#ACL_GROUP_ANONYMOUSE
self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in file_aclInfo['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group')
self.assertTrue(ACL.ACL_READ in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right')
self.assertTrue(ACL.ACL_WRITE in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right')
self.assertTrue(ACL.ACL_READ_ACP in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read_acp right')
#ACL_GROUP_CANONICAL
self.assertTrue(ACL.ACL_GROUP_CANONICAL in file_aclInfo['ACL'], 'The acl dose not contains GRPS0000000CANONICAL group')
self.assertTrue(ACL.ACL_READ in file_aclInfo['ACL'][ACL.ACL_GROUP_CANONICAL], 'The acl GRPS0000000CANONICAL group hasn\'t read right')
self.assertTrue(ACL.ACL_WRITE_ACP in file_aclInfo['ACL'][ACL.ACL_GROUP_CANONICAL], 'The acl GRPS0000000CANONICAL group hasn\'t write_acp right')
self.assertTrue(ACL.ACL_READ_ACP in file_aclInfo['ACL'][ACL.ACL_GROUP_CANONICAL], 'The acl GRPS0000000CANONICAL group hasn\'t read_acp right')
#user_id
self.assertTrue(user_id in file_aclInfo['ACL'], 'The acl dose not contains user_id group')
self.assertTrue(ACL.ACL_READ in file_aclInfo['ACL'][user_id], 'The acl user_id group hasn\'t read right')
self.assertTrue(ACL.ACL_WRITE in file_aclInfo['ACL'][user_id], 'The acl user_id group hasn\'t write right')
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
self.response = s.send(s.request(method="GET", key=None, subresource='meta'))
metaResult = json.loads(self.response.read())
self.response.close()
except SCSError, e:
self.state = RunnableState.DID_FAILED
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.emitter.emit(QtCore.SIGNAL("BucketInfoDidFailed(PyQt_PyObject,PyQt_PyObject)"), self, e.msg)
return
def test_get_bucket_meta(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
metaResult = scs.meta()
self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in metaResult['ACL'], 'The bucket:%s acl dose not contains GRPS000000ANONYMOUSE group'%self.bucket_name)
self.assertTrue(ACL.ACL_READ in metaResult['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The bucket:%s acl GRPS000000ANONYMOUSE group hasn\'t read right'%self.bucket_name)
self.assertTrue(ACL.ACL_WRITE in metaResult['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The bucket:%s acl GRPS000000ANONYMOUSE group hasn\'t write right'%self.bucket_name)
self.assertEqual(self.bucket_name, metaResult['Project'], 'The metaResult[\'Project\'] is not equal %s'%self.bucket_name)
self.assertTrue(metaResult['Owner'] is not None, 'The metaResult[\'Owner\'] is None')
self.assertTrue(metaResult['Last-Modified'] is not None, 'The metaResult[\'Last-Modified\'] is None')
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket()
self.response = s.send(s.request(key=''))
bucketJsonObj = json.loads(self.response.read())
self.response.close()
self.buckets = bucketJsonObj['Buckets']
except SCSError, e:
self.state = RunnableState.DID_FAILED
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.emitter.emit(QtCore.SIGNAL("ListBucketRunnableDidFailed(PyQt_PyObject,PyQt_PyObject)"), self, e.msg)
return
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
scsResponse = s.send(s.request(method="HEAD", key=self.key))
info = info_dict(dict(scsResponse.urllib2Response.info()))
scsResponse.close()
self.response = scsResponse
except SCSError, e:
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.state = RunnableState.DID_FAILED
self.emitter.emit(QtCore.SIGNAL("fileInfoDidFailed(PyQt_PyObject,PyQt_PyObject)"), self, e.msg)
return
def run(self):
s = SCSBucket(self.bucketName)
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
self.response = s.send(s.request(method="DELETE", key=None))
except SCSError, e:
self.state = RunnableState.DID_FAILED
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.emitter.emit(
QtCore.SIGNAL(
"DeleteBucketRunnableDidFailed(PyQt_PyObject,PyQt_PyObject)"
), self, e.msg)
return
def fun():
cloud_path = "haoc/nodes/%s" % self.path
bucket = SCSBucket(BUCKET_NAME)
# info = bucket.info(cloud_path)
total_size = 0
response = bucket[cloud_path]
CHUNK = 16 * 1024
download_size = 0
with open(local_path, 'wb') as fp:
while True:
chunk = response.read(CHUNK)
if download_size == 0:
total_size = long(response.info().dict.get(
'content-length', 0))
download_size += CHUNK
self.progress_bar.set_progress(download_size, total_size)
if not chunk:
break
fp.write(chunk)
if os.path.exists("%s.hlp" % local_path[:-4]):
response = bucket["%s.hlp" % cloud_path[:-4]]
with open("%s.hlp" % local_path[:-4], 'wb') as fp:
while True:
chunk = response.read(CHUNK)
if not chunk:
break
fp.write(chunk)
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
self.response = s.put_bucket()
self.response.close()
except SCSError, e:
self.state = RunnableState.DID_FAILED
self.emitter.emit(
QtCore.SIGNAL(
"CreateBucketDidFailed(PyQt_PyObject,PyQt_PyObject)"),
self, e.msg)
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
return
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
scsResponse = s.update_acl(self.key, self.acl)
self.response = scsResponse
except SCSError, e:
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.state = RunnableState.DID_FAILED
self.emitter.emit(
QtCore.SIGNAL(
"UpdateFileACLDidFailed(PyQt_PyObject,PyQt_PyObject)"),
self, e.msg)
return
def signin_form():
global opener, cookie, handler
sinastorage.setDefaultAppInfo(
'***********************', '*************************************')
s = SCSBucket('eggtart')
user_Agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586'
headers = {"User-Agent": user_Agent}
url0 = 'http://210.42.121.132/servlet/GenImg'
cookie = cookielib.CookieJar()
handler = urllib2.HTTPCookieProcessor(cookie)
opener = urllib2.build_opener(handler)
req0 = urllib2.Request(url0, headers=headers)
response = opener.open(req0)
s.put('picture/123.jpg', response.read())
url = s.make_url_authed('picture/123.jpg')
return render_template('signin_test.html', pic=url)
def push(dirname):
'''
把dirname文件夹下所有文件都以相同的结构同步到sae云存储中
比如，在dirname下有如下文件：
dirname/a/test.txt
dirname/test.txt
推送到sae的云存储中的文件结构就是：
/a/test.txt
/test.txt
注意空文件夹不推送
'''
akey = os.environ['SAE_STORAGE_ACCESS_KEY']
skey = os.environ['SAE_STORAGE_SECRET_KEY']
bucketname = os.environ['SAE_BUCKET_NAME']
sinastorage.setDefaultAppInfo(akey, skey)
bucket = SCSBucket(bucketname, secure=False)
fpaths = []
for root, dirs, files in os.walk(dirname):
ps = splitpath(root)
visible = True
for p in ps:
if p.startswith('.'):
visible = False
if not visible:
continue
del ps[0]
for f in files:
local_fpath = os.path.join(root, f)
ps.append(f)
fpath = pathjoin(*ps)
print('check :' + fpath)
fpaths.append(fpath)
# if not fpath.startswith(PATH_SPLITER):
# fpath =PATH_SPLITER+fpath
if file_exist(bucket, fpath):
meta = bucket.meta(fpath)
sae_sha1 = ['Content-SHA1']
fdata = open(local_fpath, 'rb').read()
if len(fdata) == 0 and meta['Size'] == 0:
pass
else:
sha1 = hashlib.sha1()
sha1.update(fdata)
if sae_sha1 != sha1.hexdigest():
bucket.putFile(fpath, local_fpath)
else:
bucket.putFile(fpath, local_fpath)
sae_files = bucket.listdir(prefix='')
for f in sae_files:
#f=(name, isPrefix, sha1, expiration_time, modify, owner, md5, content_type, size)
localf = os.path.join(dirname, f[0])
print('expecting local: ', localf)
if f[0] not in fpaths and not os.path.isfile(localf):
print('del ' + f[0])
del bucket[f[0]]
else:
print('has ' + f[0])
def fun(_time_out, _access_key, _secret_key):
sinastorage.setDefaultAppInfo(_access_key, _secret_key)
already_exist = False
all_b = SCSBucket(timeout=_time_out)
buckets_generator = all_b.list_buckets()
for bucket in buckets_generator:
if bucket[0] == BUCKET_NAME:
already_exist = True
if not already_exist:
new_bucket = SCSBucket(BUCKET_NAME, timeout=_time_out)
new_bucket.put_bucket()
new_bucket.put('haoc/', '')
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
self.response = s.send(
s.request(method="GET", key=None, subresource='meta'))
metaResult = json.loads(self.response.read())
self.response.close()
except SCSError, e:
self.state = RunnableState.DID_FAILED
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.emitter.emit(
QtCore.SIGNAL(
"BucketInfoDidFailed(PyQt_PyObject,PyQt_PyObject)"), self,
e.msg)
return
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
scsResponse = s.send(s.request(method="HEAD", key=self.key))
info = info_dict(dict(scsResponse.urllib2Response.info()))
scsResponse.close()
self.response = scsResponse
except SCSError, e:
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.state = RunnableState.DID_FAILED
self.emitter.emit(
QtCore.SIGNAL(
"fileInfoDidFailed(PyQt_PyObject,PyQt_PyObject)"), self,
e.msg)
return
def get_file():
s = SCSBucket('asdasdasdasd')
response = s['a/asdf/新建 文本文档.txt']
CHUNK = 16 * 1024
with open('111', 'wb') as fp:
while True:
chunk = response.read(CHUNK)
if not chunk: break
fp.write(chunk)
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket()
self.response = s.send(s.request(key=''))
bucketJsonObj = json.loads(self.response.read())
self.response.close()
self.buckets = bucketJsonObj['Buckets']
except SCSError, e:
self.state = RunnableState.DID_FAILED
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.emitter.emit(
QtCore.SIGNAL(
"ListBucketRunnableDidFailed(PyQt_PyObject,PyQt_PyObject)"
), self, e.msg)
return
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
m = (("prefix", smart_str(self.prefix)), ("marker", self.marker),
("max-keys", self.limit), ("delimiter", self.delimiter),
("formatter", "json"))
args = dict((str(k), str(v)) for (k, v) in m if v is not None)
self.response = s.send(s.request(key='', args=args))
self.files_generator = SCSListing.parse(self.response)
except SCSError, e:
self.state = RunnableState.DID_FAILED
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.emitter.emit(
QtCore.SIGNAL("ListDirDidFailed(PyQt_PyObject,PyQt_PyObject)"),
self, e.msg)
return
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
m = (("prefix", smart_str(self.prefix)),
("marker", self.marker),
("max-keys", self.limit),
("delimiter", self.delimiter),
("formatter","json"))
args = dict((str(k), str(v)) for (k, v) in m if v is not None)
self.response = s.send(s.request(key='', args=args))
self.files_generator = SCSListing.parse(self.response)
except SCSError, e:
self.state = RunnableState.DID_FAILED
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.emitter.emit(QtCore.SIGNAL("ListDirDidFailed(PyQt_PyObject,PyQt_PyObject)"), self, e.msg)
return
def register():
sinastorage.setDefaultAppInfo(
'***********************', '*************************************')
s = SCSBucket('sundae')
username = request.form['username']
try:
response = s[r'password/'+username+'.txt']
except BaseException, e:
password = request.form['password']
password1 = request.form['password1']
if password == password1:
password = password+username+'Sundae'
md5 = hashlib.md5()
md5.update(password.encode('utf-8'))
pwd = md5.hexdigest()
s.put(r'password/'+username+'.txt', pwd)
return '<h3>注册成功，感谢您的支持！</h3>'
else:
return '''
def fun(_time_out):
f = None
try:
ice_b = SCSBucket(BUCKET_NAME, timeout=_time_out)
response = ice_b['user/name']
f = StringIO(response.read())
return f.read()
finally:
if f is not None:
f.close()
def list_bucket_files():
s = SCSBucket('test11')
files_generator = s.listdir()#delimiter='/')
print '-----list_bucket_files---------'
print '-----detail---------'
print ('truncated : %r\n'
'marker:%r\n'
'prefix:%r\n'
'delimiter:%r\n'
'contents_quantity:%r\n'
'common_prefixes_quantity:%r\n'
'next_marker:%r'%(files_generator.truncated,
files_generator.marker,
files_generator.prefix,
files_generator.delimiter,
files_generator.contents_quantity,
files_generator.common_prefixes_quantity,
files_generator.next_marker))
print '-----file list---------'
def chuan():
sinastorage.setDefaultAppInfo(
'***********************', '*************************************')
s = SCSBucket('eggtart')
url = s.make_url_authed('picture/bj1.gif')
url0 = s.make_url_authed('picture/q3.JPG')
url1 = s.make_url_authed('picture/q4.JPG')
url2 = s.make_url_authed('picture/q7.jpg')
url3 = s.make_url_authed('picture/q2.jpg')
url4 = s.make_url_authed('picture/q1.jpg')
url5 = s.make_url_authed('picture/q8.jpg')
url6 = s.make_url_authed('picture/q5.JPG')
url7 = s.make_url_authed('picture/q0.jpg')
url8 = s.make_url_authed('picture/q6.jpg')
url9 = s.make_url_authed('picture/whu.jpg')
return render_template('cycle.html', url=url, url0=url0, url1=url1, url2=url2, url3=url3, url4=url4, url5=url5, url6=url6, url7=url7, url8=url8, whuurl=url9,)
def qing():
sinastorage.setDefaultAppInfo(
'***********************', '*************************************')
s = SCSBucket('eggtart')
url = s.make_url_authed('picture/bj1.gif')
url0 = s.make_url_authed('picture/c4.jpg')
url1 = s.make_url_authed('picture/c5.jpg')
url2 = s.make_url_authed('picture/c6.jpg')
url3 = s.make_url_authed('picture/c8.jpg')
url4 = s.make_url_authed('picture/c0.jpg')
url5 = s.make_url_authed('picture/c1.jpg')
url6 = s.make_url_authed('picture/c7.jpg')
url7 = s.make_url_authed('picture/c2.jpg')
url8 = s.make_url_authed('picture/c3.jpg')
url9 = s.make_url_authed('picture/whu.jpg')
return render_template('cycle.html', url=url, url0=url0, url1=url1, url2=url2, url3=url3, url4=url4, url5=url5, url6=url6, url7=url7, url8=url8, whuurl=url9,)
def delete(self, com, time_out):
cloud_path = "haoc/nodes/%s" % com.path
bucket = SCSBucket(BUCKET_NAME, timeout=time_out)
if cloud_path[-1] == '/':
del bucket[cloud_path]
else:
shall_do = True
if self.from_agenda:
try:
bucket["%s.nod" % cloud_path]
except sinastorage.bucket.KeyNotFound:
return
else:
info = bucket.info("%s.nod" % cloud_path)
ct = HaocUtils.utc_to_local_stamp(info['modify'])
if com.op_time < ct:
shall_do = False
if shall_do:
del bucket["%s.nod" % cloud_path]
del bucket["%s.hlp" % cloud_path]
def hello():
sinastorage.setDefaultAppInfo(
'***********************', '*************************************')
s = SCSBucket('eggtart')
pcurl = s.make_url_authed('picture/computer.png')
phoneurl = s.make_url_authed('picture/phone.png')
blogurl = s.make_url_authed('picture/blog.png')
whuurl = s.make_url_authed('picture/whu.jpg')
cycleurl = s.make_url_authed('picture/cycle.JPG')
return render_template('home.html', url1=pcurl, url2=phoneurl, url3=blogurl, url4=whuurl, url5=cycleurl)
def upload(self):
''' 普通上传 '''
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
self.fileWithCallback = FileWithCallback(self.filePath, 'rb', self.uploadCallBack)
scsResponse = s.putFileByHeaders('%s%s'%(self.prefix,os.path.basename(self.filePath)), self.fileWithCallback)
self.response = scsResponse
except SCSError, e:
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
if isinstance(e, ManualCancel): #手动取消
self.state = RunnableState.DID_CANCELED
self.response._responseBody = u'手动取消'
self.emitter.emit(QtCore.SIGNAL("fileUploadDidCanceled(PyQt_PyObject,PyQt_PyObject)"), self, e.msg)
else:
self.state = RunnableState.DID_FAILED
self.response._responseBody = e.data
self.emitter.emit(QtCore.SIGNAL("fileUploadDidFailed(PyQt_PyObject,PyQt_PyObject)"), self, e.msg)
return
def test_get_file_meta(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
metadata = {} #自定义文件属性信息
metadata['author'] = 'dage'
metadata['home'] = 'tianjin'
metadata['age'] = '18'
mimetype = 'text/plain'
scs.putFile(key=self.object_key, filePath=self.local_file_name,
metadata=metadata, mimetype=mimetype)
metaResult = scs.meta(self.object_key)
self.assertEqual(self.object_key, metaResult['File-Name'], 'The meta[\'File-Name\'] is not equals '+self.object_key)
self.assertEqual(mimetype, metaResult['Type'], 'The meta[\'Type\'] is not equals '+mimetype)
self.assertTrue(metaResult['Content-SHA1'] is not None, 'The meta[\'Content-SHA1\'] is None')
self.assertTrue(metaResult['Content-MD5'] is not None, 'The meta[\'Content-MD5\'] is None')
self.assertTrue(metaResult['Owner'] is not None, 'The meta[\'Owner\'] is None')
self.assertTrue('x-amz-meta-author' in metaResult['File-Meta'], 'File-Meta dose not contains x-amz-meta-author key')
self.assertEqual(metadata['author'], metaResult['File-Meta']['x-amz-meta-author'], 'The metaResult[\'File-Meta\'][\'x-amz-meta-author\'] value is not match')
self.assertTrue('x-amz-meta-home' in metaResult['File-Meta'], 'File-Meta dose not contains x-amz-meta-home key')
self.assertEqual(metadata['home'], metaResult['File-Meta']['x-amz-meta-home'], 'The metaResult[\'File-Meta\'][\'x-amz-meta-home\'] value is not match')
self.assertTrue('x-amz-meta-age' in metaResult['File-Meta'], 'File-Meta dose not contains x-amz-meta-age key')
self.assertEqual(metadata['age'], metaResult['File-Meta']['x-amz-meta-age'], 'The metaResult[\'File-Meta\'][\'x-amz-meta-age\'] value is not match')
def blog_signin():
username = request.form['username']
password = request.form['password']+username+'Sundae'
md5 = hashlib.md5()
md5.update(password.encode('utf-8'))
pwd = md5.hexdigest()
sinastorage.setDefaultAppInfo(
'***********************', '*************************************')
s = SCSBucket('sundae')
try:
response = s[r'password/'+username+'.txt']
except BaseException, e:
return '''
def storeImage(imgSrc):
sinastorage.setDefaultAppInfo("1cjfyo5kQPdnsI3cUc6W", "a3c139370a3509f269331930515729747573aa10")
djBucket = SCSBucket("dj-images") # not dj_images
acl = {} # Access control list
acl[ACL.ACL_GROUP_ANONYMOUSE] = [ACL.ACL_READ]
data = urllib2.urlopen(imgSrc).read()
path = imgSrc.split("/")[2] + "/"
# if '/' in file name there will be problems
filename = path + imgSrc.replace("/", "@")
scsResponse = djBucket.put(filename, data) # upload the file
stUrl = djBucket.make_url(filename) # get url of image in the storage
djBucket.update_acl(filename, acl) # set acl for the file
# save infomation to sql
try:
imgstorage.objects.get_or_create(original_url=imgSrc, storage_url=stUrl)
except:
print "insert into mysql error"
pass # solve this later
return stUrl
def tearDown(self):
scs = SCSBucket(self.bucket_name)
if scs.exist() :
try:
file_gen = scs.listdir()
for item in file_gen:
if not item[1]:
scs.delete(item[0])
scs.delete_bucket()
except Exception as err:
self.fail('empty bucket failed.Error is %s'%err)
def test_delete_file(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
content = u'this is a file content text!!'
scs.put(self.object_key, content)
try:
scs.delete(self.object_key)
except:
self.fail('Delete file is failed')
