def test_put_file_relax(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
scs.put(self.object_key, 'fileContent...fileContent...fileContent...fileContent...fileContent...')
orign_file_metaResult = scs.meta(self.object_key)
canned_acl = 'public-read-write' #快捷ACL
metadata = {} #自定义文件属性信息
metadata['author'] = 'dage'
metadata['home'] = 'tianjin'
metadata['age'] = '18'
mimetype = 'text/plain'
scs.put_relax(key=self.object_key+'_relax', sina_sha1=orign_file_metaResult['Content-SHA1'],
s_sina_length=orign_file_metaResult['Size'], acl=canned_acl,
metadata=metadata, mimetype=mimetype)
relax_file_aclInfo = scs.acl_info(self.object_key+'_relax')
self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in relax_file_aclInfo['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group')
self.assertTrue(ACL.ACL_READ in relax_file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right')
self.assertTrue(ACL.ACL_WRITE in relax_file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right')
relax_file_info = scs.info(self.object_key+'_relax')
#assert metadata
self.assertEqual(metadata['author'], relax_file_info['metadata']['author'], 'The response metadata[\'author\'] is not match')
self.assertEqual(metadata['home'], relax_file_info['metadata']['home'], 'The response metadata[\'home\'] is not match')
self.assertEqual(metadata['age'], relax_file_info['metadata']['age'], 'The response metadata[\'age\'] is not match')
#assert content-type
self.assertEqual(mimetype, relax_file_info['headers']['content-type'], 'The response content-type is not match')
def test_update_file_meta(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
content = u'this is a file content text!!'
scs.put(self.object_key, content)
canned_acl = 'public-read-write' #快捷ACL
metadata = {} #自定义文件属性信息
metadata['author'] = 'dage'
metadata['home'] = 'tianjin'
metadata['age'] = '18'
mimetype = 'text/plain'
scs.update_meta(self.object_key, metadata=metadata, acl=canned_acl, mimetype=mimetype)
file_info = scs.info(self.object_key)
#assert metadata
self.assertEqual(metadata['author'], file_info['metadata']['author'], 'The response metadata[\'author\'] is not match')
self.assertEqual(metadata['home'], file_info['metadata']['home'], 'The response metadata[\'home\'] is not match')
self.assertEqual(metadata['age'], file_info['metadata']['age'], 'The response metadata[\'age\'] is not match')
#assert content-type
# print file_info['headers']
#TODO:服务器问题 self.assertEqual(mimetype, file_info['headers']['content-type'], 'The response content-type is not match')
#acl
file_aclInfo = scs.acl_info(self.object_key)
self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in file_aclInfo['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group')
self.assertTrue(ACL.ACL_READ in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right')
self.assertTrue(ACL.ACL_WRITE in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right')
def test_update_file_acl(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
content = u'this is a file content text!!'
canned_acl = 'private' #快捷ACL
scs.put(key=self.object_key, data=content, acl=canned_acl)
#设置ACL
acl = {}
acl[ACL.ACL_GROUP_ANONYMOUSE] = [ACL.ACL_READ, ACL.ACL_WRITE, ACL.ACL_READ_ACP]
acl[ACL.ACL_GROUP_CANONICAL] = [ACL.ACL_READ_ACP,ACL.ACL_WRITE_ACP, ACL.ACL_READ]
user_id = 'SINA0000001001AABBCC'
acl[user_id] = [ACL.ACL_WRITE, ACL.ACL_READ]
scs.update_acl(self.object_key, acl)
#assert
file_aclInfo = scs.acl_info(self.object_key)
#ACL_GROUP_ANONYMOUSE
self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in file_aclInfo['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group')
self.assertTrue(ACL.ACL_READ in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right')
self.assertTrue(ACL.ACL_WRITE in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right')
self.assertTrue(ACL.ACL_READ_ACP in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read_acp right')
#ACL_GROUP_CANONICAL
self.assertTrue(ACL.ACL_GROUP_CANONICAL in file_aclInfo['ACL'], 'The acl dose not contains GRPS0000000CANONICAL group')
self.assertTrue(ACL.ACL_READ in file_aclInfo['ACL'][ACL.ACL_GROUP_CANONICAL], 'The acl GRPS0000000CANONICAL group hasn\'t read right')
self.assertTrue(ACL.ACL_WRITE_ACP in file_aclInfo['ACL'][ACL.ACL_GROUP_CANONICAL], 'The acl GRPS0000000CANONICAL group hasn\'t write_acp right')
self.assertTrue(ACL.ACL_READ_ACP in file_aclInfo['ACL'][ACL.ACL_GROUP_CANONICAL], 'The acl GRPS0000000CANONICAL group hasn\'t read_acp right')
#user_id
self.assertTrue(user_id in file_aclInfo['ACL'], 'The acl dose not contains user_id group')
self.assertTrue(ACL.ACL_READ in file_aclInfo['ACL'][user_id], 'The acl user_id group hasn\'t read right')
self.assertTrue(ACL.ACL_WRITE in file_aclInfo['ACL'][user_id], 'The acl user_id group hasn\'t write right')
def fun(_time_out, _access_key, _secret_key):
sinastorage.setDefaultAppInfo(_access_key, _secret_key)
already_exist = False
all_b = SCSBucket(timeout=_time_out)
buckets_generator = all_b.list_buckets()
for bucket in buckets_generator:
if bucket[0] == BUCKET_NAME:
already_exist = True
if not already_exist:
new_bucket = SCSBucket(BUCKET_NAME, timeout=_time_out)
new_bucket.put_bucket()
new_bucket.put('haoc/', '')
def test_delete_file(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
content = u'this is a file content text!!'
scs.put(self.object_key, content)
try:
scs.delete(self.object_key)
except:
self.fail('Delete file is failed')
def test_file_acl_info(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
content = u'this is a file content text!!'
canned_acl = 'public-read-write' #快捷ACL
scs.put(key=self.object_key, data=content, acl=canned_acl)
#acl
file_aclInfo = scs.acl_info(self.object_key)
self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in file_aclInfo['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group')
self.assertTrue(ACL.ACL_READ in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right')
self.assertTrue(ACL.ACL_WRITE in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right')
def signin_form():
global opener, cookie, handler
sinastorage.setDefaultAppInfo(
'***********************', '*************************************')
s = SCSBucket('eggtart')
user_Agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586'
headers = {"User-Agent": user_Agent}
url0 = 'http://210.42.121.132/servlet/GenImg'
cookie = cookielib.CookieJar()
handler = urllib2.HTTPCookieProcessor(cookie)
opener = urllib2.build_opener(handler)
req0 = urllib2.Request(url0, headers=headers)
response = opener.open(req0)
s.put('picture/123.jpg', response.read())
url = s.make_url_authed('picture/123.jpg')
return render_template('signin_test.html', pic=url)
def test_make_url_authed(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('private')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
content = u'this is a file content text!!'
canned_acl = 'private' #快捷ACL
scs.put(key=self.object_key, data=content, acl=canned_acl)
url = scs.make_url_authed(key=self.object_key,expire=datetime.timedelta(minutes=5))
#下载文件
# import urllib2
# response = urllib2.urlopen(url)
from sinastorage.compat import urllib
response = urllib.request.urlopen(url)
self.assertEqual(len(content), int(response.info()['Content-Length']), 'The response header Content-Length is not match')
def register():
sinastorage.setDefaultAppInfo(
'***********************', '*************************************')
s = SCSBucket('sundae')
username = request.form['username']
try:
response = s[r'password/'+username+'.txt']
except BaseException, e:
password = request.form['password']
password1 = request.form['password1']
if password == password1:
password = password+username+'Sundae'
md5 = hashlib.md5()
md5.update(password.encode('utf-8'))
pwd = md5.hexdigest()
s.put(r'password/'+username+'.txt', pwd)
return '<h3>注册成功，感谢您的支持！</h3>'
else:
return '''
def test_list_bucket_files(self):
scs = SCSBucket(self.bucket_name)
scs.put_bucket()
content = u'this is a file content text!!'
scs.put(self.object_key,content)
file_gen = scs.listdir()
self.assertTrue(file_gen is not None, 'List bucket files result is None')
for item in file_gen:
#(name, isPrefix, sha1, expiration_time, modify, owner, md5, content_type, size)
self.assertTrue(item[0] is not None, 'file name is None')
self.assertTrue(item[1] is not None, 'file isPrefix is None')
self.assertTrue(item[2] is not None, 'file sha1 is None')
# self.assertIsNotNone(item[3], 'file expiration_time is None')
self.assertTrue(item[4] is not None, 'file modify is None')
self.assertTrue(item[5] is not None, 'file owner is None')
self.assertTrue(item[6] is not None, 'file md5 is None')
self.assertTrue(item[7] is not None, 'file content_type is None')
self.assertTrue(item[8] is not None, 'file size is None')
def test_put_content(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
content = u'this is a file content text!!'
scs.put(self.object_key, content)
scsResponse = scs.get(self.object_key)
CHUNK = 16 * 1024
file_content = ''
from sinastorage.vendored import six
while True:
chunk = scsResponse.read(CHUNK)
if not chunk: break
if isinstance(chunk, six.text_type):
file_content += chunk
else:
file_content += chunk.decode("utf-8")
self.assertEqual(content, file_content, 'The uploaded file content is not match local.%s'%file_content)
def test_put_content_with_conditions(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
content = u'this is a file content text!!' #文件内容
canned_acl = 'public-read-write' #快捷ACL
metadata = {} #自定义文件属性信息
metadata['author'] = 'dage'
metadata['home'] = 'tianjin'
metadata['age'] = '18'
mimetype = 'text/plain'
scs.put(self.object_key, content, acl=canned_acl, metadata=metadata, mimetype=mimetype)
scsResponse = scs.get(self.object_key)
CHUNK = 16 * 1024
file_content = ''
from sinastorage.vendored import six
while True:
chunk = scsResponse.read(CHUNK)
if not chunk: break
if isinstance(chunk, six.text_type):
file_content += chunk
else:
file_content += chunk.decode("utf-8")
#assert file_content
self.assertEqual(content, file_content, 'The uploaded file content is not metch local.%s'%file_content)
#assert metadata
self.assertEqual(metadata['author'], scsResponse.responseHeaders['x-amz-meta-author'], 'The response metadata[\'author\'] is not match')
self.assertEqual(metadata['home'], scsResponse.responseHeaders['x-amz-meta-home'], 'The response metadata[\'home\'] is not match')
self.assertEqual(metadata['age'], scsResponse.responseHeaders['x-amz-meta-age'], 'The response metadata[\'age\'] is not match')
#assert content-type
self.assertEqual(mimetype, scsResponse.responseHeaders['content-type'], 'The response content-type is not match')
#assert acl
aclResult = scs.acl_info(self.object_key)
self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in aclResult['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group')
self.assertTrue(ACL.ACL_READ in aclResult['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right')
self.assertTrue(ACL.ACL_WRITE in aclResult['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right')
def obj_upload(data, fileTag):
""" Upload photo by object. """
sinastorage.setDefaultAppInfo(API_KEY, API_SECRET)
s = SCSBucket('sharephotos')
path = time.ctime().replace(' ', '_')
# if '/' in file name there will be problems
filename = fileTag[:6] + '_' + path + '.jpg' # use jpg temporary
scs_response = s.put(filename, data)
s.update_acl(filename, acl)
url = s.make_url(filename) # get url of image in the storage
return url
def create(self, com):
if com.path[-1] != '/':
self.new_item_start.emit(os.path.basename(com.path))
cloud_path = "haoc/nodes/%s" % com.path
bucket = SCSBucket(BUCKET_NAME)
local_path = "%s/data/%s/nodes/%s" % (
HaocUtils.get_root_path(), HaocUtils.Config.get_ak(), com.path)
if cloud_path[-1] == '/':
bucket.put(cloud_path, '')
else:
shall_upload = True
if not os.path.exists("%s.nod" % local_path):
print "File not found:%s" % local_path
return
lt = int(os.path.getmtime("%s.nod" % local_path))
if self.from_agenda:
try:
bucket["%s.nod" % cloud_path]
except sinastorage.bucket.KeyNotFound:
pass
else:
info = bucket.info("%s.nod" % cloud_path)
ct = int(info.get('metadata').get('lmt'))
if lt < ct:
shall_upload = False
if shall_upload:
bucket.putFile("%s.nod" % cloud_path, "%s.nod" % local_path,
self.call_back)
bucket.update_meta("%s.nod" % cloud_path, {'lmt': str(lt)})
bucket.update_meta("%s.nod" % cloud_path,
{'hver': hou.applicationVersionString()})
if os.path.exists("%s.hlp" % local_path):
bucket.putFile("%s.hlp" % cloud_path,
"%s.hlp" % local_path)
else:
open("%s.nod" % local_path, 'w').close()
def url_upload(imgSrc='http://www.w3school.com.cn/i/site_photoref.jpg'):
""" Upload photo by url. """
sinastorage.setDefaultAppInfo(API_KEY, API_SECRET)
s = SCSBucket('sharephotos')
data = urllib2.urlopen(imgSrc).read()
path = imgSrc.split('/')[2] + '/%s__' % time.ctime()
# if '/' in file name there will be problems
filename = path + imgSrc.replace('/', '_')
scs_response = s.put(filename, data)
s.update_acl(filename, acl)
url = s.make_url(filename) # get url of image in the storage
return url
def test_info_file(self):
scs = SCSBucket(self.bucket_name)
try:
scs.put_bucket('public-read-write')
except:
self.fail('create bucket:%s is failed'%self.bucket_name)
content = u'this is a file content text!!' #文件内容
canned_acl = 'public-read-write' #快捷ACL
metadata = {} #自定义文件属性信息
metadata['author'] = 'dage'
metadata['home'] = 'tianjin'
metadata['age'] = '18'
mimetype = 'text/plain'
scs.put(self.object_key, content, acl=canned_acl, metadata=metadata, mimetype=mimetype)
file_info = scs.info(self.object_key)
#assert metadata
self.assertEqual(metadata['author'], file_info['metadata']['author'], 'The response metadata[\'author\'] is not match')
self.assertEqual(metadata['home'], file_info['metadata']['home'], 'The response metadata[\'home\'] is not match')
self.assertEqual(metadata['age'], file_info['metadata']['age'], 'The response metadata[\'age\'] is not match')
#assert content-type
self.assertEqual(mimetype, file_info['headers']['content-type'], 'The response content-type is not match')
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
scsResponse = s.put(self.key,'')
self.response = scsResponse
self.response.close()
except SCSError, e:
self.state = RunnableState.DID_FAILED
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.emitter.emit(QtCore.SIGNAL("CreateFolderDidFailed(PyQt_PyObject,PyQt_PyObject)"), self, e.msg)
return
def run(self):
try:
self.mutex.lock()
self.state = RunnableState.RUNNING
s = SCSBucket(self.bucketName)
scsResponse = s.put(self.key, '')
self.response = scsResponse
self.response.close()
except SCSError, e:
self.state = RunnableState.DID_FAILED
self.response = SCSResponse(e.urllib2Request, e.urllib2Response)
self.response._responseBody = e.data
self.emitter.emit(
QtCore.SIGNAL(
"CreateFolderDidFailed(PyQt_PyObject,PyQt_PyObject)"),
self, e.msg)
return
def storeImage(imgSrc):
sinastorage.setDefaultAppInfo("1cjfyo5kQPdnsI3cUc6W", "a3c139370a3509f269331930515729747573aa10")
djBucket = SCSBucket("dj-images") # not dj_images
acl = {} # Access control list
acl[ACL.ACL_GROUP_ANONYMOUSE] = [ACL.ACL_READ]
data = urllib2.urlopen(imgSrc).read()
path = imgSrc.split("/")[2] + "/"
# if '/' in file name there will be problems
filename = path + imgSrc.replace("/", "@")
scsResponse = djBucket.put(filename, data) # upload the file
stUrl = djBucket.make_url(filename) # get url of image in the storage
djBucket.update_acl(filename, acl) # set acl for the file
# save infomation to sql
try:
imgstorage.objects.get_or_create(original_url=imgSrc, storage_url=stUrl)
except:
print "insert into mysql error"
pass # solve this later
return stUrl
import sinastorage
D = Downloader()
result = D('http://www.csindex.com.cn/sseportal/ps/zhs/hqjt/csi/show_zsgz.js')
decoded_result = result.decode('gbk').replace('var ', '')
decoded_result = decoded_result.replace('"', '')
result_lists = decoded_result.split('\r\n')
d = {}
for row in result_lists:
if len(row) == 0:
continue
e = row.split('=')
d[e[0]] = e[1]
count_str = ''
line_str = None
sinastorage.setDefaultAppInfo('16wn9n74cnZJFjffXU3K',
'50706013e9a806c252aafb03ad2a3b51174e36ad')
s = SCSBucket('saepcs', secure=False)
for r in range(11):
line_str = d['zsgz00']
for e in range(9):
count_str = 'zsgz' + str(r + 1) + str(e)
line_str = line_str + '\t' + d[count_str]
line_str = line_str + '\r\n'
print(line_str)
with open('indexvalue_temp.txt', 'a', encoding='utf-8', newline='') as f:
f.write(line_str)
#s = SCSBucket('saepcs', secure=False)
scsResponse = s.put('stock/test.txt', line_str)
class Uploader():
def __init__(self, host, accesskey, secretkey, bucket_name, remote_dir):
self.host = host
self.bucket_name = bucket_name
self.remote_dir = remote_dir
sinastorage.setDefaultAppInfo(accesskey, secretkey)
self.handler = SCSBucket(self.bucket_name)
def do(self, url, filepath_list, is_file=True):
remote_list = {}
sub_dir = self.get_subdir(url)
if filepath_list:
for _url, filepath, filename in filepath_list:
if not filepath:
continue
filename = filename if filename else str(time.time()) + ".html"
print("#INFO: uploader ", filename, sub_dir)
sys.stdout.flush()
try:
remote_link = self.upload_file(filepath, filename, sub_dir,
is_file)
remote_list[_url] = remote_link
except Exception as e:
print("#ERR: upload filed failed, %s, %s", filename,
str(e))
continue
return remote_list
@my_utils.time_limit(5)
def upload_file(self, local_path, local_name, sub_dir="", is_file=True):
if not local_path or not local_name:
return ""
if sub_dir:
sub_dir = sub_dir + "/"
remote_path = self.remote_dir + "/" + sub_dir + local_name
ret = self._upload_file(remote_path, local_path, is_file)
self.set_acl(remote_path)
return ret
def _upload_file(self, remote_path, local_path, is_file=True):
if is_file:
ret = self.handler.putFile(remote_path, local_path)
else:
ret = self.handler.put(remote_path, local_path)
return self.host + "/" + self.bucket_name + "/" + remote_path if ret else ""
def set_acl(self, remote_path):
acl = {}
acl[ACL.ACL_GROUP_ANONYMOUSE] = [ACL.ACL_READ]
acl[ACL.ACL_GROUP_CANONICAL] = [ACL.ACL_READ_ACP, ACL.ACL_WRITE_ACP]
self.handler.update_acl(remote_path, acl)
def get_subdir(self, url):
url_p = urlparse.urlparse(url)
return url_p.hostname.replace(".", "_")
def fun(_time_out, _name):
ice_b = SCSBucket(BUCKET_NAME, timeout=_time_out)
ice_b.put('user/name', _name)
timestamp = int(calendar.timegm(timestamp.utctimetuple()))
length_file = int(r[8])
yield fuse.Direntry(name=tmp_name, st_mode=stat.S_IFREG | 0755, st_nlink=1, st_size=length_file,st_atime=timestamp,st_mtime=timestamp,st_ctime=timestamp)
def mkdir(self, path, mode):
if path.startswith('/'):
path = path[1:]
if not path.endswith('/'):
path = path+'/'
try:
finfo = s.info(path)
return -errno.ENOTEMPTY
except Exception, e:
pass
s.put(path, u'')
def unlink(self, path):
if path.startswith('/'):
path = path[1:]
del s[path]
class XmpFile(object):
def __init__(self, path, flags, *mode):
(tmp_f, tmp_path) = tempfile.mkstemp(prefix='scs')
os.close(tmp_f)
try:
download(path, tmp_path)
except Exception, e:
pass
def put_content():
content = u'this is a file content text!!'
s = SCSBucket('create-a-bucket11')
print s.put('sss.txt',content)
