def fun(_time_out): if self.lis is not None: self.lis = [] bucket = SCSBucket(BUCKET_NAME, timeout=_time_out) items_generator = bucket.listdir(prefix='haoc/nodes/', limit=10000) access_key = HaocUtils.Config.get_ak() for item in items_generator: common_name = item[0][11:] local_name = "%s/data/%s/nodes/%s" % ( HaocUtils.get_root_path(), access_key, common_name) if self.lis is not None: if common_name[-1] == '/' or common_name[-4:] == '.nod': self.lis.append(common_name) if common_name[-1] == '/': if not os.path.exists(local_name): os.makedirs(local_name) elif common_name[-4:]: if os.path.exists(local_name): lt = int(os.path.getmtime("%s.nod" % local_name[:-4])) ct = int( bucket.info( "%s.nod" % item[0][:-4]).get('metadata').get('lmt')) # Compare local file modified time with cloud file modified time if lt < ct: open(local_name, "w").close() if is_manual_sync: print "Updated: %s" % common_name else: if not os.path.exists(os.path.dirname(local_name)): os.makedirs(os.path.dirname(local_name)) open(local_name, "w").close() if is_manual_sync: print "Created: %s" % common_name
def listFile(bucketName): Files = [] s = SCSBucket(bucketName) files = s.listdir() for i in files: #i = (name, isPrefix, sha1, expiration_time, modify, owner, md5, content_type, size) Files.append(i) return Files
def push(dirname): ''' 把dirname文件夹下所有文件都以相同的结构同步到sae云存储中 比如,在dirname下有如下文件: dirname/a/test.txt dirname/test.txt 推送到sae的云存储中的文件结构就是: /a/test.txt /test.txt 注意空文件夹不推送 ''' akey = os.environ['SAE_STORAGE_ACCESS_KEY'] skey = os.environ['SAE_STORAGE_SECRET_KEY'] bucketname = os.environ['SAE_BUCKET_NAME'] sinastorage.setDefaultAppInfo(akey, skey) bucket = SCSBucket(bucketname, secure=False) fpaths = [] for root, dirs, files in os.walk(dirname): ps = splitpath(root) visible = True for p in ps: if p.startswith('.'): visible = False if not visible: continue del ps[0] for f in files: local_fpath = os.path.join(root, f) ps.append(f) fpath = pathjoin(*ps) print('check :' + fpath) fpaths.append(fpath) # if not fpath.startswith(PATH_SPLITER): # fpath =PATH_SPLITER+fpath if file_exist(bucket, fpath): meta = bucket.meta(fpath) sae_sha1 = ['Content-SHA1'] fdata = open(local_fpath, 'rb').read() if len(fdata) == 0 and meta['Size'] == 0: pass else: sha1 = hashlib.sha1() sha1.update(fdata) if sae_sha1 != sha1.hexdigest(): bucket.putFile(fpath, local_fpath) else: bucket.putFile(fpath, local_fpath) sae_files = bucket.listdir(prefix='') for f in sae_files: #f=(name, isPrefix, sha1, expiration_time, modify, owner, md5, content_type, size) localf = os.path.join(dirname, f[0]) print('expecting local: ', localf) if f[0] not in fpaths and not os.path.isfile(localf): print('del ' + f[0]) del bucket[f[0]] else: print('has ' + f[0])
def tearDown(self): scs = SCSBucket(self.bucket_name) if scs.exist() : try: file_gen = scs.listdir() for item in file_gen: if not item[1]: scs.delete(item[0]) scs.delete_bucket() except Exception as err: self.fail('empty bucket failed.Error is %s'%err)
def list_bucket_files(): s = SCSBucket('test11') files_generator = s.listdir()#delimiter='/') print '-----list_bucket_files---------' print '-----detail---------' print ('truncated : %r\n' 'marker:%r\n' 'prefix:%r\n' 'delimiter:%r\n' 'contents_quantity:%r\n' 'common_prefixes_quantity:%r\n' 'next_marker:%r'%(files_generator.truncated, files_generator.marker, files_generator.prefix, files_generator.delimiter, files_generator.contents_quantity, files_generator.common_prefixes_quantity, files_generator.next_marker)) print '-----file list---------'
def test_list_bucket_files(self): scs = SCSBucket(self.bucket_name) scs.put_bucket() content = u'this is a file content text!!' scs.put(self.object_key,content) file_gen = scs.listdir() self.assertTrue(file_gen is not None, 'List bucket files result is None') for item in file_gen: #(name, isPrefix, sha1, expiration_time, modify, owner, md5, content_type, size) self.assertTrue(item[0] is not None, 'file name is None') self.assertTrue(item[1] is not None, 'file isPrefix is None') self.assertTrue(item[2] is not None, 'file sha1 is None') # self.assertIsNotNone(item[3], 'file expiration_time is None') self.assertTrue(item[4] is not None, 'file modify is None') self.assertTrue(item[5] is not None, 'file owner is None') self.assertTrue(item[6] is not None, 'file md5 is None') self.assertTrue(item[7] is not None, 'file content_type is None') self.assertTrue(item[8] is not None, 'file size is None')