def fun(_time_out):
if self.lis is not None:
self.lis = []
bucket = SCSBucket(BUCKET_NAME, timeout=_time_out)
items_generator = bucket.listdir(prefix='haoc/nodes/', limit=10000)
access_key = HaocUtils.Config.get_ak()
for item in items_generator:
common_name = item[0][11:]
local_name = "%s/data/%s/nodes/%s" % (
HaocUtils.get_root_path(), access_key, common_name)
if self.lis is not None:
if common_name[-1] == '/' or common_name[-4:] == '.nod':
self.lis.append(common_name)
if common_name[-1] == '/':
if not os.path.exists(local_name):
os.makedirs(local_name)
elif common_name[-4:]:
if os.path.exists(local_name):
lt = int(os.path.getmtime("%s.nod" % local_name[:-4]))
ct = int(
bucket.info(
"%s.nod" %
item[0][:-4]).get('metadata').get('lmt'))
# Compare local file modified time with cloud file modified time
if lt < ct:
open(local_name, "w").close()
if is_manual_sync:
print "Updated: %s" % common_name
else:
if not os.path.exists(os.path.dirname(local_name)):
os.makedirs(os.path.dirname(local_name))
open(local_name, "w").close()
if is_manual_sync:
print "Created: %s" % common_name
def listFile(bucketName): Files = [] s = SCSBucket(bucketName) files = s.listdir() for i in files: #i = (name, isPrefix, sha1, expiration_time, modify, owner, md5, content_type, size) Files.append(i) return Files
def push(dirname):
'''
把dirname文件夹下所有文件都以相同的结构同步到sae云存储中
比如，在dirname下有如下文件：
dirname/a/test.txt
dirname/test.txt
推送到sae的云存储中的文件结构就是：
/a/test.txt
/test.txt
注意空文件夹不推送
'''
akey = os.environ['SAE_STORAGE_ACCESS_KEY']
skey = os.environ['SAE_STORAGE_SECRET_KEY']
bucketname = os.environ['SAE_BUCKET_NAME']
sinastorage.setDefaultAppInfo(akey, skey)
bucket = SCSBucket(bucketname, secure=False)
fpaths = []
for root, dirs, files in os.walk(dirname):
ps = splitpath(root)
visible = True
for p in ps:
if p.startswith('.'):
visible = False
if not visible:
continue
del ps[0]
for f in files:
local_fpath = os.path.join(root, f)
ps.append(f)
fpath = pathjoin(*ps)
print('check :' + fpath)
fpaths.append(fpath)
# if not fpath.startswith(PATH_SPLITER):
# fpath =PATH_SPLITER+fpath
if file_exist(bucket, fpath):
meta = bucket.meta(fpath)
sae_sha1 = ['Content-SHA1']
fdata = open(local_fpath, 'rb').read()
if len(fdata) == 0 and meta['Size'] == 0:
pass
else:
sha1 = hashlib.sha1()
sha1.update(fdata)
if sae_sha1 != sha1.hexdigest():
bucket.putFile(fpath, local_fpath)
else:
bucket.putFile(fpath, local_fpath)
sae_files = bucket.listdir(prefix='')
for f in sae_files:
#f=(name, isPrefix, sha1, expiration_time, modify, owner, md5, content_type, size)
localf = os.path.join(dirname, f[0])
print('expecting local: ', localf)
if f[0] not in fpaths and not os.path.isfile(localf):
print('del ' + f[0])
del bucket[f[0]]
else:
print('has ' + f[0])
def tearDown(self):
scs = SCSBucket(self.bucket_name)
if scs.exist() :
try:
file_gen = scs.listdir()
for item in file_gen:
if not item[1]:
scs.delete(item[0])
scs.delete_bucket()
except Exception as err:
self.fail('empty bucket failed.Error is %s'%err)
def list_bucket_files():
s = SCSBucket('test11')
files_generator = s.listdir()#delimiter='/')
print '-----list_bucket_files---------'
print '-----detail---------'
print ('truncated : %r\n'
'marker:%r\n'
'prefix:%r\n'
'delimiter:%r\n'
'contents_quantity:%r\n'
'common_prefixes_quantity:%r\n'
'next_marker:%r'%(files_generator.truncated,
files_generator.marker,
files_generator.prefix,
files_generator.delimiter,
files_generator.contents_quantity,
files_generator.common_prefixes_quantity,
files_generator.next_marker))
print '-----file list---------'
def test_list_bucket_files(self):
scs = SCSBucket(self.bucket_name)
scs.put_bucket()
content = u'this is a file content text!!'
scs.put(self.object_key,content)
file_gen = scs.listdir()
self.assertTrue(file_gen is not None, 'List bucket files result is None')
for item in file_gen:
#(name, isPrefix, sha1, expiration_time, modify, owner, md5, content_type, size)
self.assertTrue(item[0] is not None, 'file name is None')
self.assertTrue(item[1] is not None, 'file isPrefix is None')
self.assertTrue(item[2] is not None, 'file sha1 is None')
# self.assertIsNotNone(item[3], 'file expiration_time is None')
self.assertTrue(item[4] is not None, 'file modify is None')
self.assertTrue(item[5] is not None, 'file owner is None')
self.assertTrue(item[6] is not None, 'file md5 is None')
self.assertTrue(item[7] is not None, 'file content_type is None')
self.assertTrue(item[8] is not None, 'file size is None')
