def test_put_file_relax(self): scs = SCSBucket(self.bucket_name) try: scs.put_bucket('public-read-write') except: self.fail('create bucket:%s is failed'%self.bucket_name) scs.put(self.object_key, 'fileContent...fileContent...fileContent...fileContent...fileContent...') orign_file_metaResult = scs.meta(self.object_key) canned_acl = 'public-read-write' #快捷ACL metadata = {} #自定义文件属性信息 metadata['author'] = 'dage' metadata['home'] = 'tianjin' metadata['age'] = '18' mimetype = 'text/plain' scs.put_relax(key=self.object_key+'_relax', sina_sha1=orign_file_metaResult['Content-SHA1'], s_sina_length=orign_file_metaResult['Size'], acl=canned_acl, metadata=metadata, mimetype=mimetype) relax_file_aclInfo = scs.acl_info(self.object_key+'_relax') self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in relax_file_aclInfo['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group') self.assertTrue(ACL.ACL_READ in relax_file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right') self.assertTrue(ACL.ACL_WRITE in relax_file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right') relax_file_info = scs.info(self.object_key+'_relax') #assert metadata self.assertEqual(metadata['author'], relax_file_info['metadata']['author'], 'The response metadata[\'author\'] is not match') self.assertEqual(metadata['home'], relax_file_info['metadata']['home'], 'The response metadata[\'home\'] is not match') self.assertEqual(metadata['age'], relax_file_info['metadata']['age'], 'The response metadata[\'age\'] is not match') #assert content-type self.assertEqual(mimetype, relax_file_info['headers']['content-type'], 'The response content-type is not match')
def test_update_file_meta(self): scs = SCSBucket(self.bucket_name) try: scs.put_bucket('public-read-write') except: self.fail('create bucket:%s is failed'%self.bucket_name) content = u'this is a file content text!!' scs.put(self.object_key, content) canned_acl = 'public-read-write' #快捷ACL metadata = {} #自定义文件属性信息 metadata['author'] = 'dage' metadata['home'] = 'tianjin' metadata['age'] = '18' mimetype = 'text/plain' scs.update_meta(self.object_key, metadata=metadata, acl=canned_acl, mimetype=mimetype) file_info = scs.info(self.object_key) #assert metadata self.assertEqual(metadata['author'], file_info['metadata']['author'], 'The response metadata[\'author\'] is not match') self.assertEqual(metadata['home'], file_info['metadata']['home'], 'The response metadata[\'home\'] is not match') self.assertEqual(metadata['age'], file_info['metadata']['age'], 'The response metadata[\'age\'] is not match') #assert content-type # print file_info['headers'] #TODO:服务器问题 self.assertEqual(mimetype, file_info['headers']['content-type'], 'The response content-type is not match') #acl file_aclInfo = scs.acl_info(self.object_key) self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in file_aclInfo['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group') self.assertTrue(ACL.ACL_READ in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right') self.assertTrue(ACL.ACL_WRITE in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right')
def test_update_file_acl(self): scs = SCSBucket(self.bucket_name) try: scs.put_bucket('public-read-write') except: self.fail('create bucket:%s is failed'%self.bucket_name) content = u'this is a file content text!!' canned_acl = 'private' #快捷ACL scs.put(key=self.object_key, data=content, acl=canned_acl) #设置ACL acl = {} acl[ACL.ACL_GROUP_ANONYMOUSE] = [ACL.ACL_READ, ACL.ACL_WRITE, ACL.ACL_READ_ACP] acl[ACL.ACL_GROUP_CANONICAL] = [ACL.ACL_READ_ACP,ACL.ACL_WRITE_ACP, ACL.ACL_READ] user_id = 'SINA0000001001AABBCC' acl[user_id] = [ACL.ACL_WRITE, ACL.ACL_READ] scs.update_acl(self.object_key, acl) #assert file_aclInfo = scs.acl_info(self.object_key) #ACL_GROUP_ANONYMOUSE self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in file_aclInfo['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group') self.assertTrue(ACL.ACL_READ in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right') self.assertTrue(ACL.ACL_WRITE in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right') self.assertTrue(ACL.ACL_READ_ACP in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read_acp right') #ACL_GROUP_CANONICAL self.assertTrue(ACL.ACL_GROUP_CANONICAL in file_aclInfo['ACL'], 'The acl dose not contains GRPS0000000CANONICAL group') self.assertTrue(ACL.ACL_READ in file_aclInfo['ACL'][ACL.ACL_GROUP_CANONICAL], 'The acl GRPS0000000CANONICAL group hasn\'t read right') self.assertTrue(ACL.ACL_WRITE_ACP in file_aclInfo['ACL'][ACL.ACL_GROUP_CANONICAL], 'The acl GRPS0000000CANONICAL group hasn\'t write_acp right') self.assertTrue(ACL.ACL_READ_ACP in file_aclInfo['ACL'][ACL.ACL_GROUP_CANONICAL], 'The acl GRPS0000000CANONICAL group hasn\'t read_acp right') #user_id self.assertTrue(user_id in file_aclInfo['ACL'], 'The acl dose not contains user_id group') self.assertTrue(ACL.ACL_READ in file_aclInfo['ACL'][user_id], 'The acl user_id group hasn\'t read right') self.assertTrue(ACL.ACL_WRITE in file_aclInfo['ACL'][user_id], 'The acl user_id group hasn\'t write right')
def fun(_time_out, _access_key, _secret_key): sinastorage.setDefaultAppInfo(_access_key, _secret_key) already_exist = False all_b = SCSBucket(timeout=_time_out) buckets_generator = all_b.list_buckets() for bucket in buckets_generator: if bucket[0] == BUCKET_NAME: already_exist = True if not already_exist: new_bucket = SCSBucket(BUCKET_NAME, timeout=_time_out) new_bucket.put_bucket() new_bucket.put('haoc/', '')
def test_delete_file(self): scs = SCSBucket(self.bucket_name) try: scs.put_bucket('public-read-write') except: self.fail('create bucket:%s is failed'%self.bucket_name) content = u'this is a file content text!!' scs.put(self.object_key, content) try: scs.delete(self.object_key) except: self.fail('Delete file is failed')
def test_file_acl_info(self): scs = SCSBucket(self.bucket_name) try: scs.put_bucket('public-read-write') except: self.fail('create bucket:%s is failed'%self.bucket_name) content = u'this is a file content text!!' canned_acl = 'public-read-write' #快捷ACL scs.put(key=self.object_key, data=content, acl=canned_acl) #acl file_aclInfo = scs.acl_info(self.object_key) self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in file_aclInfo['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group') self.assertTrue(ACL.ACL_READ in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right') self.assertTrue(ACL.ACL_WRITE in file_aclInfo['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right')
def signin_form(): global opener, cookie, handler sinastorage.setDefaultAppInfo( '***********************', '*************************************') s = SCSBucket('eggtart') user_Agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586' headers = {"User-Agent": user_Agent} url0 = 'http://210.42.121.132/servlet/GenImg' cookie = cookielib.CookieJar() handler = urllib2.HTTPCookieProcessor(cookie) opener = urllib2.build_opener(handler) req0 = urllib2.Request(url0, headers=headers) response = opener.open(req0) s.put('picture/123.jpg', response.read()) url = s.make_url_authed('picture/123.jpg') return render_template('signin_test.html', pic=url)
def test_make_url_authed(self): scs = SCSBucket(self.bucket_name) try: scs.put_bucket('private') except: self.fail('create bucket:%s is failed'%self.bucket_name) content = u'this is a file content text!!' canned_acl = 'private' #快捷ACL scs.put(key=self.object_key, data=content, acl=canned_acl) url = scs.make_url_authed(key=self.object_key,expire=datetime.timedelta(minutes=5)) #下载文件 # import urllib2 # response = urllib2.urlopen(url) from sinastorage.compat import urllib response = urllib.request.urlopen(url) self.assertEqual(len(content), int(response.info()['Content-Length']), 'The response header Content-Length is not match')
def register(): sinastorage.setDefaultAppInfo( '***********************', '*************************************') s = SCSBucket('sundae') username = request.form['username'] try: response = s[r'password/'+username+'.txt'] except BaseException, e: password = request.form['password'] password1 = request.form['password1'] if password == password1: password = password+username+'Sundae' md5 = hashlib.md5() md5.update(password.encode('utf-8')) pwd = md5.hexdigest() s.put(r'password/'+username+'.txt', pwd) return '<h3>注册成功,感谢您的支持!</h3>' else: return '''
def test_list_bucket_files(self): scs = SCSBucket(self.bucket_name) scs.put_bucket() content = u'this is a file content text!!' scs.put(self.object_key,content) file_gen = scs.listdir() self.assertTrue(file_gen is not None, 'List bucket files result is None') for item in file_gen: #(name, isPrefix, sha1, expiration_time, modify, owner, md5, content_type, size) self.assertTrue(item[0] is not None, 'file name is None') self.assertTrue(item[1] is not None, 'file isPrefix is None') self.assertTrue(item[2] is not None, 'file sha1 is None') # self.assertIsNotNone(item[3], 'file expiration_time is None') self.assertTrue(item[4] is not None, 'file modify is None') self.assertTrue(item[5] is not None, 'file owner is None') self.assertTrue(item[6] is not None, 'file md5 is None') self.assertTrue(item[7] is not None, 'file content_type is None') self.assertTrue(item[8] is not None, 'file size is None')
def test_put_content(self): scs = SCSBucket(self.bucket_name) try: scs.put_bucket('public-read-write') except: self.fail('create bucket:%s is failed'%self.bucket_name) content = u'this is a file content text!!' scs.put(self.object_key, content) scsResponse = scs.get(self.object_key) CHUNK = 16 * 1024 file_content = '' from sinastorage.vendored import six while True: chunk = scsResponse.read(CHUNK) if not chunk: break if isinstance(chunk, six.text_type): file_content += chunk else: file_content += chunk.decode("utf-8") self.assertEqual(content, file_content, 'The uploaded file content is not match local.%s'%file_content)
def test_put_content_with_conditions(self): scs = SCSBucket(self.bucket_name) try: scs.put_bucket('public-read-write') except: self.fail('create bucket:%s is failed'%self.bucket_name) content = u'this is a file content text!!' #文件内容 canned_acl = 'public-read-write' #快捷ACL metadata = {} #自定义文件属性信息 metadata['author'] = 'dage' metadata['home'] = 'tianjin' metadata['age'] = '18' mimetype = 'text/plain' scs.put(self.object_key, content, acl=canned_acl, metadata=metadata, mimetype=mimetype) scsResponse = scs.get(self.object_key) CHUNK = 16 * 1024 file_content = '' from sinastorage.vendored import six while True: chunk = scsResponse.read(CHUNK) if not chunk: break if isinstance(chunk, six.text_type): file_content += chunk else: file_content += chunk.decode("utf-8") #assert file_content self.assertEqual(content, file_content, 'The uploaded file content is not metch local.%s'%file_content) #assert metadata self.assertEqual(metadata['author'], scsResponse.responseHeaders['x-amz-meta-author'], 'The response metadata[\'author\'] is not match') self.assertEqual(metadata['home'], scsResponse.responseHeaders['x-amz-meta-home'], 'The response metadata[\'home\'] is not match') self.assertEqual(metadata['age'], scsResponse.responseHeaders['x-amz-meta-age'], 'The response metadata[\'age\'] is not match') #assert content-type self.assertEqual(mimetype, scsResponse.responseHeaders['content-type'], 'The response content-type is not match') #assert acl aclResult = scs.acl_info(self.object_key) self.assertTrue(ACL.ACL_GROUP_ANONYMOUSE in aclResult['ACL'], 'The acl dose not contains GRPS000000ANONYMOUSE group') self.assertTrue(ACL.ACL_READ in aclResult['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t read right') self.assertTrue(ACL.ACL_WRITE in aclResult['ACL'][ACL.ACL_GROUP_ANONYMOUSE], 'The acl GRPS000000ANONYMOUSE group hasn\'t write right')
def obj_upload(data, fileTag): """ Upload photo by object. """ sinastorage.setDefaultAppInfo(API_KEY, API_SECRET) s = SCSBucket('sharephotos') path = time.ctime().replace(' ', '_') # if '/' in file name there will be problems filename = fileTag[:6] + '_' + path + '.jpg' # use jpg temporary scs_response = s.put(filename, data) s.update_acl(filename, acl) url = s.make_url(filename) # get url of image in the storage return url
def create(self, com): if com.path[-1] != '/': self.new_item_start.emit(os.path.basename(com.path)) cloud_path = "haoc/nodes/%s" % com.path bucket = SCSBucket(BUCKET_NAME) local_path = "%s/data/%s/nodes/%s" % ( HaocUtils.get_root_path(), HaocUtils.Config.get_ak(), com.path) if cloud_path[-1] == '/': bucket.put(cloud_path, '') else: shall_upload = True if not os.path.exists("%s.nod" % local_path): print "File not found:%s" % local_path return lt = int(os.path.getmtime("%s.nod" % local_path)) if self.from_agenda: try: bucket["%s.nod" % cloud_path] except sinastorage.bucket.KeyNotFound: pass else: info = bucket.info("%s.nod" % cloud_path) ct = int(info.get('metadata').get('lmt')) if lt < ct: shall_upload = False if shall_upload: bucket.putFile("%s.nod" % cloud_path, "%s.nod" % local_path, self.call_back) bucket.update_meta("%s.nod" % cloud_path, {'lmt': str(lt)}) bucket.update_meta("%s.nod" % cloud_path, {'hver': hou.applicationVersionString()}) if os.path.exists("%s.hlp" % local_path): bucket.putFile("%s.hlp" % cloud_path, "%s.hlp" % local_path) else: open("%s.nod" % local_path, 'w').close()
def url_upload(imgSrc='http://www.w3school.com.cn/i/site_photoref.jpg'): """ Upload photo by url. """ sinastorage.setDefaultAppInfo(API_KEY, API_SECRET) s = SCSBucket('sharephotos') data = urllib2.urlopen(imgSrc).read() path = imgSrc.split('/')[2] + '/%s__' % time.ctime() # if '/' in file name there will be problems filename = path + imgSrc.replace('/', '_') scs_response = s.put(filename, data) s.update_acl(filename, acl) url = s.make_url(filename) # get url of image in the storage return url
def test_info_file(self): scs = SCSBucket(self.bucket_name) try: scs.put_bucket('public-read-write') except: self.fail('create bucket:%s is failed'%self.bucket_name) content = u'this is a file content text!!' #文件内容 canned_acl = 'public-read-write' #快捷ACL metadata = {} #自定义文件属性信息 metadata['author'] = 'dage' metadata['home'] = 'tianjin' metadata['age'] = '18' mimetype = 'text/plain' scs.put(self.object_key, content, acl=canned_acl, metadata=metadata, mimetype=mimetype) file_info = scs.info(self.object_key) #assert metadata self.assertEqual(metadata['author'], file_info['metadata']['author'], 'The response metadata[\'author\'] is not match') self.assertEqual(metadata['home'], file_info['metadata']['home'], 'The response metadata[\'home\'] is not match') self.assertEqual(metadata['age'], file_info['metadata']['age'], 'The response metadata[\'age\'] is not match') #assert content-type self.assertEqual(mimetype, file_info['headers']['content-type'], 'The response content-type is not match')
def run(self): try: self.mutex.lock() self.state = RunnableState.RUNNING s = SCSBucket(self.bucketName) scsResponse = s.put(self.key,'') self.response = scsResponse self.response.close() except SCSError, e: self.state = RunnableState.DID_FAILED self.response = SCSResponse(e.urllib2Request, e.urllib2Response) self.response._responseBody = e.data self.emitter.emit(QtCore.SIGNAL("CreateFolderDidFailed(PyQt_PyObject,PyQt_PyObject)"), self, e.msg) return
def run(self): try: self.mutex.lock() self.state = RunnableState.RUNNING s = SCSBucket(self.bucketName) scsResponse = s.put(self.key, '') self.response = scsResponse self.response.close() except SCSError, e: self.state = RunnableState.DID_FAILED self.response = SCSResponse(e.urllib2Request, e.urllib2Response) self.response._responseBody = e.data self.emitter.emit( QtCore.SIGNAL( "CreateFolderDidFailed(PyQt_PyObject,PyQt_PyObject)"), self, e.msg) return
def storeImage(imgSrc): sinastorage.setDefaultAppInfo("1cjfyo5kQPdnsI3cUc6W", "a3c139370a3509f269331930515729747573aa10") djBucket = SCSBucket("dj-images") # not dj_images acl = {} # Access control list acl[ACL.ACL_GROUP_ANONYMOUSE] = [ACL.ACL_READ] data = urllib2.urlopen(imgSrc).read() path = imgSrc.split("/")[2] + "/" # if '/' in file name there will be problems filename = path + imgSrc.replace("/", "@") scsResponse = djBucket.put(filename, data) # upload the file stUrl = djBucket.make_url(filename) # get url of image in the storage djBucket.update_acl(filename, acl) # set acl for the file # save infomation to sql try: imgstorage.objects.get_or_create(original_url=imgSrc, storage_url=stUrl) except: print "insert into mysql error" pass # solve this later return stUrl
import sinastorage D = Downloader() result = D('http://www.csindex.com.cn/sseportal/ps/zhs/hqjt/csi/show_zsgz.js') decoded_result = result.decode('gbk').replace('var ', '') decoded_result = decoded_result.replace('"', '') result_lists = decoded_result.split('\r\n') d = {} for row in result_lists: if len(row) == 0: continue e = row.split('=') d[e[0]] = e[1] count_str = '' line_str = None sinastorage.setDefaultAppInfo('16wn9n74cnZJFjffXU3K', '50706013e9a806c252aafb03ad2a3b51174e36ad') s = SCSBucket('saepcs', secure=False) for r in range(11): line_str = d['zsgz00'] for e in range(9): count_str = 'zsgz' + str(r + 1) + str(e) line_str = line_str + '\t' + d[count_str] line_str = line_str + '\r\n' print(line_str) with open('indexvalue_temp.txt', 'a', encoding='utf-8', newline='') as f: f.write(line_str) #s = SCSBucket('saepcs', secure=False) scsResponse = s.put('stock/test.txt', line_str)
class Uploader(): def __init__(self, host, accesskey, secretkey, bucket_name, remote_dir): self.host = host self.bucket_name = bucket_name self.remote_dir = remote_dir sinastorage.setDefaultAppInfo(accesskey, secretkey) self.handler = SCSBucket(self.bucket_name) def do(self, url, filepath_list, is_file=True): remote_list = {} sub_dir = self.get_subdir(url) if filepath_list: for _url, filepath, filename in filepath_list: if not filepath: continue filename = filename if filename else str(time.time()) + ".html" print("#INFO: uploader ", filename, sub_dir) sys.stdout.flush() try: remote_link = self.upload_file(filepath, filename, sub_dir, is_file) remote_list[_url] = remote_link except Exception as e: print("#ERR: upload filed failed, %s, %s", filename, str(e)) continue return remote_list @my_utils.time_limit(5) def upload_file(self, local_path, local_name, sub_dir="", is_file=True): if not local_path or not local_name: return "" if sub_dir: sub_dir = sub_dir + "/" remote_path = self.remote_dir + "/" + sub_dir + local_name ret = self._upload_file(remote_path, local_path, is_file) self.set_acl(remote_path) return ret def _upload_file(self, remote_path, local_path, is_file=True): if is_file: ret = self.handler.putFile(remote_path, local_path) else: ret = self.handler.put(remote_path, local_path) return self.host + "/" + self.bucket_name + "/" + remote_path if ret else "" def set_acl(self, remote_path): acl = {} acl[ACL.ACL_GROUP_ANONYMOUSE] = [ACL.ACL_READ] acl[ACL.ACL_GROUP_CANONICAL] = [ACL.ACL_READ_ACP, ACL.ACL_WRITE_ACP] self.handler.update_acl(remote_path, acl) def get_subdir(self, url): url_p = urlparse.urlparse(url) return url_p.hostname.replace(".", "_")
def fun(_time_out, _name): ice_b = SCSBucket(BUCKET_NAME, timeout=_time_out) ice_b.put('user/name', _name)
timestamp = int(calendar.timegm(timestamp.utctimetuple())) length_file = int(r[8]) yield fuse.Direntry(name=tmp_name, st_mode=stat.S_IFREG | 0755, st_nlink=1, st_size=length_file,st_atime=timestamp,st_mtime=timestamp,st_ctime=timestamp) def mkdir(self, path, mode): if path.startswith('/'): path = path[1:] if not path.endswith('/'): path = path+'/' try: finfo = s.info(path) return -errno.ENOTEMPTY except Exception, e: pass s.put(path, u'') def unlink(self, path): if path.startswith('/'): path = path[1:] del s[path] class XmpFile(object): def __init__(self, path, flags, *mode): (tmp_f, tmp_path) = tempfile.mkstemp(prefix='scs') os.close(tmp_f) try: download(path, tmp_path) except Exception, e: pass
def put_content(): content = u'this is a file content text!!' s = SCSBucket('create-a-bucket11') print s.put('sss.txt',content)